1 //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file declares the SelectionDAG class, and transitively defines the 10 // SDNode class and subclasses. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_SELECTIONDAG_H 15 #define LLVM_CODEGEN_SELECTIONDAG_H 16 17 #include "llvm/ADT/APFloat.h" 18 #include "llvm/ADT/APInt.h" 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/DenseSet.h" 22 #include "llvm/ADT/FoldingSet.h" 23 #include "llvm/ADT/SmallVector.h" 24 #include "llvm/ADT/StringMap.h" 25 #include "llvm/ADT/ilist.h" 26 #include "llvm/ADT/iterator.h" 27 #include "llvm/ADT/iterator_range.h" 28 #include "llvm/CodeGen/DAGCombine.h" 29 #include "llvm/CodeGen/ISDOpcodes.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineMemOperand.h" 32 #include "llvm/CodeGen/SelectionDAGNodes.h" 33 #include "llvm/CodeGen/ValueTypes.h" 34 #include "llvm/IR/DebugLoc.h" 35 #include "llvm/IR/Metadata.h" 36 #include "llvm/Support/Allocator.h" 37 #include "llvm/Support/ArrayRecycler.h" 38 #include "llvm/Support/CodeGen.h" 39 #include "llvm/Support/ErrorHandling.h" 40 #include "llvm/Support/MachineValueType.h" 41 #include "llvm/Support/RecyclingAllocator.h" 42 #include <cassert> 43 #include <cstdint> 44 #include <functional> 45 #include <map> 46 #include <string> 47 #include <tuple> 48 #include <utility> 49 #include <vector> 50 51 namespace llvm { 52 53 class DIExpression; 54 class DILabel; 55 class DIVariable; 56 class Function; 57 class Pass; 58 class Type; 59 template <class GraphType> struct GraphTraits; 60 template <typename T, unsigned int N> class SmallSetVector; 61 template <typename T, typename Enable> struct FoldingSetTrait; 62 class AAResults; 63 class BlockAddress; 64 class BlockFrequencyInfo; 65 class Constant; 66 class ConstantFP; 67 class ConstantInt; 68 class DataLayout; 69 struct fltSemantics; 70 class FunctionLoweringInfo; 71 class FunctionVarLocs; 72 class GlobalValue; 73 struct KnownBits; 74 class LegacyDivergenceAnalysis; 75 class LLVMContext; 76 class MachineBasicBlock; 77 class MachineConstantPoolValue; 78 class MCSymbol; 79 class OptimizationRemarkEmitter; 80 class ProfileSummaryInfo; 81 class SDDbgValue; 82 class SDDbgOperand; 83 class SDDbgLabel; 84 class SelectionDAG; 85 class SelectionDAGTargetInfo; 86 class TargetLibraryInfo; 87 class TargetLowering; 88 class TargetMachine; 89 class TargetSubtargetInfo; 90 class Value; 91 92 class SDVTListNode : public FoldingSetNode { 93 friend struct FoldingSetTrait<SDVTListNode>; 94 95 /// A reference to an Interned FoldingSetNodeID for this node. 96 /// The Allocator in SelectionDAG holds the data. 97 /// SDVTList contains all types which are frequently accessed in SelectionDAG. 98 /// The size of this list is not expected to be big so it won't introduce 99 /// a memory penalty. 100 FoldingSetNodeIDRef FastID; 101 const EVT *VTs; 102 unsigned int NumVTs; 103 /// The hash value for SDVTList is fixed, so cache it to avoid 104 /// hash calculation. 105 unsigned HashValue; 106 107 public: 108 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : 109 FastID(ID), VTs(VT), NumVTs(Num) { 110 HashValue = ID.ComputeHash(); 111 } 112 113 SDVTList getSDVTList() { 114 SDVTList result = {VTs, NumVTs}; 115 return result; 116 } 117 }; 118 119 /// Specialize FoldingSetTrait for SDVTListNode 120 /// to avoid computing temp FoldingSetNodeID and hash value. 121 template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> { 122 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { 123 ID = X.FastID; 124 } 125 126 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, 127 unsigned IDHash, FoldingSetNodeID &TempID) { 128 if (X.HashValue != IDHash) 129 return false; 130 return ID == X.FastID; 131 } 132 133 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) { 134 return X.HashValue; 135 } 136 }; 137 138 template <> struct ilist_alloc_traits<SDNode> { 139 static void deleteNode(SDNode *) { 140 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!"); 141 } 142 }; 143 144 /// Keeps track of dbg_value information through SDISel. We do 145 /// not build SDNodes for these so as not to perturb the generated code; 146 /// instead the info is kept off to the side in this structure. Each SDNode may 147 /// have one or more associated dbg_value entries. This information is kept in 148 /// DbgValMap. 149 /// Byval parameters are handled separately because they don't use alloca's, 150 /// which busts the normal mechanism. There is good reason for handling all 151 /// parameters separately: they may not have code generated for them, they 152 /// should always go at the beginning of the function regardless of other code 153 /// motion, and debug info for them is potentially useful even if the parameter 154 /// is unused. Right now only byval parameters are handled separately. 155 class SDDbgInfo { 156 BumpPtrAllocator Alloc; 157 SmallVector<SDDbgValue*, 32> DbgValues; 158 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; 159 SmallVector<SDDbgLabel*, 4> DbgLabels; 160 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>; 161 DbgValMapType DbgValMap; 162 163 public: 164 SDDbgInfo() = default; 165 SDDbgInfo(const SDDbgInfo &) = delete; 166 SDDbgInfo &operator=(const SDDbgInfo &) = delete; 167 168 void add(SDDbgValue *V, bool isParameter); 169 170 void add(SDDbgLabel *L) { DbgLabels.push_back(L); } 171 172 /// Invalidate all DbgValues attached to the node and remove 173 /// it from the Node-to-DbgValues map. 174 void erase(const SDNode *Node); 175 176 void clear() { 177 DbgValMap.clear(); 178 DbgValues.clear(); 179 ByvalParmDbgValues.clear(); 180 DbgLabels.clear(); 181 Alloc.Reset(); 182 } 183 184 BumpPtrAllocator &getAlloc() { return Alloc; } 185 186 bool empty() const { 187 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); 188 } 189 190 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const { 191 auto I = DbgValMap.find(Node); 192 if (I != DbgValMap.end()) 193 return I->second; 194 return ArrayRef<SDDbgValue*>(); 195 } 196 197 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator; 198 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator; 199 200 DbgIterator DbgBegin() { return DbgValues.begin(); } 201 DbgIterator DbgEnd() { return DbgValues.end(); } 202 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); } 203 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); } 204 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); } 205 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); } 206 }; 207 208 void checkForCycles(const SelectionDAG *DAG, bool force = false); 209 210 /// This is used to represent a portion of an LLVM function in a low-level 211 /// Data Dependence DAG representation suitable for instruction selection. 212 /// This DAG is constructed as the first step of instruction selection in order 213 /// to allow implementation of machine specific optimizations 214 /// and code simplifications. 215 /// 216 /// The representation used by the SelectionDAG is a target-independent 217 /// representation, which has some similarities to the GCC RTL representation, 218 /// but is significantly more simple, powerful, and is a graph form instead of a 219 /// linear form. 220 /// 221 class SelectionDAG { 222 const TargetMachine &TM; 223 const SelectionDAGTargetInfo *TSI = nullptr; 224 const TargetLowering *TLI = nullptr; 225 const TargetLibraryInfo *LibInfo = nullptr; 226 const FunctionVarLocs *FnVarLocs = nullptr; 227 MachineFunction *MF; 228 Pass *SDAGISelPass = nullptr; 229 LLVMContext *Context; 230 CodeGenOpt::Level OptLevel; 231 232 LegacyDivergenceAnalysis * DA = nullptr; 233 FunctionLoweringInfo * FLI = nullptr; 234 235 /// The function-level optimization remark emitter. Used to emit remarks 236 /// whenever manipulating the DAG. 237 OptimizationRemarkEmitter *ORE; 238 239 ProfileSummaryInfo *PSI = nullptr; 240 BlockFrequencyInfo *BFI = nullptr; 241 242 /// List of non-single value types. 243 FoldingSet<SDVTListNode> VTListMap; 244 245 /// Pool allocation for misc. objects that are created once per SelectionDAG. 246 BumpPtrAllocator Allocator; 247 248 /// The starting token. 249 SDNode EntryNode; 250 251 /// The root of the entire DAG. 252 SDValue Root; 253 254 /// A linked list of nodes in the current DAG. 255 ilist<SDNode> AllNodes; 256 257 /// The AllocatorType for allocating SDNodes. We use 258 /// pool allocation with recycling. 259 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode, 260 sizeof(LargestSDNode), 261 alignof(MostAlignedSDNode)>; 262 263 /// Pool allocation for nodes. 264 NodeAllocatorType NodeAllocator; 265 266 /// This structure is used to memoize nodes, automatically performing 267 /// CSE with existing nodes when a duplicate is requested. 268 FoldingSet<SDNode> CSEMap; 269 270 /// Pool allocation for machine-opcode SDNode operands. 271 BumpPtrAllocator OperandAllocator; 272 ArrayRecycler<SDUse> OperandRecycler; 273 274 /// Tracks dbg_value and dbg_label information through SDISel. 275 SDDbgInfo *DbgInfo; 276 277 using CallSiteInfo = MachineFunction::CallSiteInfo; 278 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; 279 280 struct NodeExtraInfo { 281 CallSiteInfo CSInfo; 282 MDNode *HeapAllocSite = nullptr; 283 MDNode *PCSections = nullptr; 284 bool NoMerge = false; 285 }; 286 /// Out-of-line extra information for SDNodes. 287 DenseMap<const SDNode *, NodeExtraInfo> SDEI; 288 289 /// PersistentId counter to be used when inserting the next 290 /// SDNode to this SelectionDAG. We do not place that under 291 /// `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` intentionally because 292 /// it adds unneeded complexity without noticeable 293 /// benefits (see discussion with @thakis in D120714). 294 uint16_t NextPersistentId = 0; 295 296 public: 297 /// Clients of various APIs that cause global effects on 298 /// the DAG can optionally implement this interface. This allows the clients 299 /// to handle the various sorts of updates that happen. 300 /// 301 /// A DAGUpdateListener automatically registers itself with DAG when it is 302 /// constructed, and removes itself when destroyed in RAII fashion. 303 struct DAGUpdateListener { 304 DAGUpdateListener *const Next; 305 SelectionDAG &DAG; 306 307 explicit DAGUpdateListener(SelectionDAG &D) 308 : Next(D.UpdateListeners), DAG(D) { 309 DAG.UpdateListeners = this; 310 } 311 312 virtual ~DAGUpdateListener() { 313 assert(DAG.UpdateListeners == this && 314 "DAGUpdateListeners must be destroyed in LIFO order"); 315 DAG.UpdateListeners = Next; 316 } 317 318 /// The node N that was deleted and, if E is not null, an 319 /// equivalent node E that replaced it. 320 virtual void NodeDeleted(SDNode *N, SDNode *E); 321 322 /// The node N that was updated. 323 virtual void NodeUpdated(SDNode *N); 324 325 /// The node N that was inserted. 326 virtual void NodeInserted(SDNode *N); 327 }; 328 329 struct DAGNodeDeletedListener : public DAGUpdateListener { 330 std::function<void(SDNode *, SDNode *)> Callback; 331 332 DAGNodeDeletedListener(SelectionDAG &DAG, 333 std::function<void(SDNode *, SDNode *)> Callback) 334 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} 335 336 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } 337 338 private: 339 virtual void anchor(); 340 }; 341 342 struct DAGNodeInsertedListener : public DAGUpdateListener { 343 std::function<void(SDNode *)> Callback; 344 345 DAGNodeInsertedListener(SelectionDAG &DAG, 346 std::function<void(SDNode *)> Callback) 347 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} 348 349 void NodeInserted(SDNode *N) override { Callback(N); } 350 351 private: 352 virtual void anchor(); 353 }; 354 355 /// Help to insert SDNodeFlags automatically in transforming. Use 356 /// RAII to save and resume flags in current scope. 357 class FlagInserter { 358 SelectionDAG &DAG; 359 SDNodeFlags Flags; 360 FlagInserter *LastInserter; 361 362 public: 363 FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags) 364 : DAG(SDAG), Flags(Flags), 365 LastInserter(SDAG.getFlagInserter()) { 366 SDAG.setFlagInserter(this); 367 } 368 FlagInserter(SelectionDAG &SDAG, SDNode *N) 369 : FlagInserter(SDAG, N->getFlags()) {} 370 371 FlagInserter(const FlagInserter &) = delete; 372 FlagInserter &operator=(const FlagInserter &) = delete; 373 ~FlagInserter() { DAG.setFlagInserter(LastInserter); } 374 375 SDNodeFlags getFlags() const { return Flags; } 376 }; 377 378 /// When true, additional steps are taken to 379 /// ensure that getConstant() and similar functions return DAG nodes that 380 /// have legal types. This is important after type legalization since 381 /// any illegally typed nodes generated after this point will not experience 382 /// type legalization. 383 bool NewNodesMustHaveLegalTypes = false; 384 385 private: 386 /// DAGUpdateListener is a friend so it can manipulate the listener stack. 387 friend struct DAGUpdateListener; 388 389 /// Linked list of registered DAGUpdateListener instances. 390 /// This stack is maintained by DAGUpdateListener RAII. 391 DAGUpdateListener *UpdateListeners = nullptr; 392 393 /// Implementation of setSubgraphColor. 394 /// Return whether we had to truncate the search. 395 bool setSubgraphColorHelper(SDNode *N, const char *Color, 396 DenseSet<SDNode *> &visited, 397 int level, bool &printed); 398 399 template <typename SDNodeT, typename... ArgTypes> 400 SDNodeT *newSDNode(ArgTypes &&... Args) { 401 return new (NodeAllocator.template Allocate<SDNodeT>()) 402 SDNodeT(std::forward<ArgTypes>(Args)...); 403 } 404 405 /// Build a synthetic SDNodeT with the given args and extract its subclass 406 /// data as an integer (e.g. for use in a folding set). 407 /// 408 /// The args to this function are the same as the args to SDNodeT's 409 /// constructor, except the second arg (assumed to be a const DebugLoc&) is 410 /// omitted. 411 template <typename SDNodeT, typename... ArgTypes> 412 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder, 413 ArgTypes &&... Args) { 414 // The compiler can reduce this expression to a constant iff we pass an 415 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing 416 // on the subclass data. 417 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...) 418 .getRawSubclassData(); 419 } 420 421 template <typename SDNodeTy> 422 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order, 423 SDVTList VTs, EVT MemoryVT, 424 MachineMemOperand *MMO) { 425 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO) 426 .getRawSubclassData(); 427 } 428 429 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals); 430 431 void removeOperands(SDNode *Node) { 432 if (!Node->OperandList) 433 return; 434 OperandRecycler.deallocate( 435 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands), 436 Node->OperandList); 437 Node->NumOperands = 0; 438 Node->OperandList = nullptr; 439 } 440 void CreateTopologicalOrder(std::vector<SDNode*>& Order); 441 442 public: 443 // Maximum depth for recursive analysis such as computeKnownBits, etc. 444 static constexpr unsigned MaxRecursionDepth = 6; 445 446 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); 447 SelectionDAG(const SelectionDAG &) = delete; 448 SelectionDAG &operator=(const SelectionDAG &) = delete; 449 ~SelectionDAG(); 450 451 /// Prepare this SelectionDAG to process code in the given MachineFunction. 452 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, 453 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 454 LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, 455 BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs); 456 457 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { 458 FLI = FuncInfo; 459 } 460 461 /// Clear state and free memory necessary to make this 462 /// SelectionDAG ready to process a new block. 463 void clear(); 464 465 MachineFunction &getMachineFunction() const { return *MF; } 466 const Pass *getPass() const { return SDAGISelPass; } 467 468 const DataLayout &getDataLayout() const { return MF->getDataLayout(); } 469 const TargetMachine &getTarget() const { return TM; } 470 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } 471 template <typename STC> const STC &getSubtarget() const { 472 return MF->getSubtarget<STC>(); 473 } 474 const TargetLowering &getTargetLoweringInfo() const { return *TLI; } 475 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } 476 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } 477 const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } 478 /// Returns the result of the AssignmentTrackingAnalysis pass if it's 479 /// available, otherwise return nullptr. 480 const FunctionVarLocs *getFunctionVarLocs() const { return FnVarLocs; } 481 LLVMContext *getContext() const { return Context; } 482 OptimizationRemarkEmitter &getORE() const { return *ORE; } 483 ProfileSummaryInfo *getPSI() const { return PSI; } 484 BlockFrequencyInfo *getBFI() const { return BFI; } 485 486 FlagInserter *getFlagInserter() { return Inserter; } 487 void setFlagInserter(FlagInserter *FI) { Inserter = FI; } 488 489 /// Just dump dot graph to a user-provided path and title. 490 /// This doesn't open the dot viewer program and 491 /// helps visualization when outside debugging session. 492 /// FileName expects absolute path. If provided 493 /// without any path separators then the file 494 /// will be created in the current directory. 495 /// Error will be emitted if the path is insane. 496 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 497 LLVM_DUMP_METHOD void dumpDotGraph(const Twine &FileName, const Twine &Title); 498 #endif 499 500 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. 501 void viewGraph(const std::string &Title); 502 void viewGraph(); 503 504 #if LLVM_ENABLE_ABI_BREAKING_CHECKS 505 std::map<const SDNode *, std::string> NodeGraphAttrs; 506 #endif 507 508 /// Clear all previously defined node graph attributes. 509 /// Intended to be used from a debugging tool (eg. gdb). 510 void clearGraphAttrs(); 511 512 /// Set graph attributes for a node. (eg. "color=red".) 513 void setGraphAttrs(const SDNode *N, const char *Attrs); 514 515 /// Get graph attributes for a node. (eg. "color=red".) 516 /// Used from getNodeAttributes. 517 std::string getGraphAttrs(const SDNode *N) const; 518 519 /// Convenience for setting node color attribute. 520 void setGraphColor(const SDNode *N, const char *Color); 521 522 /// Convenience for setting subgraph color attribute. 523 void setSubgraphColor(SDNode *N, const char *Color); 524 525 using allnodes_const_iterator = ilist<SDNode>::const_iterator; 526 527 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); } 528 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); } 529 530 using allnodes_iterator = ilist<SDNode>::iterator; 531 532 allnodes_iterator allnodes_begin() { return AllNodes.begin(); } 533 allnodes_iterator allnodes_end() { return AllNodes.end(); } 534 535 ilist<SDNode>::size_type allnodes_size() const { 536 return AllNodes.size(); 537 } 538 539 iterator_range<allnodes_iterator> allnodes() { 540 return make_range(allnodes_begin(), allnodes_end()); 541 } 542 iterator_range<allnodes_const_iterator> allnodes() const { 543 return make_range(allnodes_begin(), allnodes_end()); 544 } 545 546 /// Return the root tag of the SelectionDAG. 547 const SDValue &getRoot() const { return Root; } 548 549 /// Return the token chain corresponding to the entry of the function. 550 SDValue getEntryNode() const { 551 return SDValue(const_cast<SDNode *>(&EntryNode), 0); 552 } 553 554 /// Set the current root tag of the SelectionDAG. 555 /// 556 const SDValue &setRoot(SDValue N) { 557 assert((!N.getNode() || N.getValueType() == MVT::Other) && 558 "DAG root value is not a chain!"); 559 if (N.getNode()) 560 checkForCycles(N.getNode(), this); 561 Root = N; 562 if (N.getNode()) 563 checkForCycles(this); 564 return Root; 565 } 566 567 #ifndef NDEBUG 568 void VerifyDAGDivergence(); 569 #endif 570 571 /// This iterates over the nodes in the SelectionDAG, folding 572 /// certain types of nodes together, or eliminating superfluous nodes. The 573 /// Level argument controls whether Combine is allowed to produce nodes and 574 /// types that are illegal on the target. 575 void Combine(CombineLevel Level, AAResults *AA, 576 CodeGenOpt::Level OptLevel); 577 578 /// This transforms the SelectionDAG into a SelectionDAG that 579 /// only uses types natively supported by the target. 580 /// Returns "true" if it made any changes. 581 /// 582 /// Note that this is an involved process that may invalidate pointers into 583 /// the graph. 584 bool LegalizeTypes(); 585 586 /// This transforms the SelectionDAG into a SelectionDAG that is 587 /// compatible with the target instruction selector, as indicated by the 588 /// TargetLowering object. 589 /// 590 /// Note that this is an involved process that may invalidate pointers into 591 /// the graph. 592 void Legalize(); 593 594 /// Transforms a SelectionDAG node and any operands to it into a node 595 /// that is compatible with the target instruction selector, as indicated by 596 /// the TargetLowering object. 597 /// 598 /// \returns true if \c N is a valid, legal node after calling this. 599 /// 600 /// This essentially runs a single recursive walk of the \c Legalize process 601 /// over the given node (and its operands). This can be used to incrementally 602 /// legalize the DAG. All of the nodes which are directly replaced, 603 /// potentially including N, are added to the output parameter \c 604 /// UpdatedNodes so that the delta to the DAG can be understood by the 605 /// caller. 606 /// 607 /// When this returns false, N has been legalized in a way that make the 608 /// pointer passed in no longer valid. It may have even been deleted from the 609 /// DAG, and so it shouldn't be used further. When this returns true, the 610 /// N passed in is a legal node, and can be immediately processed as such. 611 /// This may still have done some work on the DAG, and will still populate 612 /// UpdatedNodes with any new nodes replacing those originally in the DAG. 613 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes); 614 615 /// This transforms the SelectionDAG into a SelectionDAG 616 /// that only uses vector math operations supported by the target. This is 617 /// necessary as a separate step from Legalize because unrolling a vector 618 /// operation can introduce illegal types, which requires running 619 /// LegalizeTypes again. 620 /// 621 /// This returns true if it made any changes; in that case, LegalizeTypes 622 /// is called again before Legalize. 623 /// 624 /// Note that this is an involved process that may invalidate pointers into 625 /// the graph. 626 bool LegalizeVectors(); 627 628 /// This method deletes all unreachable nodes in the SelectionDAG. 629 void RemoveDeadNodes(); 630 631 /// Remove the specified node from the system. This node must 632 /// have no referrers. 633 void DeleteNode(SDNode *N); 634 635 /// Return an SDVTList that represents the list of values specified. 636 SDVTList getVTList(EVT VT); 637 SDVTList getVTList(EVT VT1, EVT VT2); 638 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3); 639 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4); 640 SDVTList getVTList(ArrayRef<EVT> VTs); 641 642 //===--------------------------------------------------------------------===// 643 // Node creation methods. 644 645 /// Create a ConstantSDNode wrapping a constant value. 646 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 647 /// 648 /// If only legal types can be produced, this does the necessary 649 /// transformations (e.g., if the vector element type is illegal). 650 /// @{ 651 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 652 bool isTarget = false, bool isOpaque = false); 653 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 654 bool isTarget = false, bool isOpaque = false); 655 656 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false, 657 bool IsOpaque = false) { 658 return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT, 659 IsTarget, IsOpaque); 660 } 661 662 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 663 bool isTarget = false, bool isOpaque = false); 664 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, 665 bool isTarget = false); 666 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, 667 bool LegalTypes = true); 668 SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, 669 bool isTarget = false); 670 671 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, 672 bool isOpaque = false) { 673 return getConstant(Val, DL, VT, true, isOpaque); 674 } 675 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT, 676 bool isOpaque = false) { 677 return getConstant(Val, DL, VT, true, isOpaque); 678 } 679 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 680 bool isOpaque = false) { 681 return getConstant(Val, DL, VT, true, isOpaque); 682 } 683 684 /// Create a true or false constant of type \p VT using the target's 685 /// BooleanContent for type \p OpVT. 686 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT); 687 /// @} 688 689 /// Create a ConstantFPSDNode wrapping a constant value. 690 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 691 /// 692 /// If only legal types can be produced, this does the necessary 693 /// transformations (e.g., if the vector element type is illegal). 694 /// The forms that take a double should only be used for simple constants 695 /// that can be exactly represented in VT. No checks are made. 696 /// @{ 697 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, 698 bool isTarget = false); 699 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT, 700 bool isTarget = false); 701 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT, 702 bool isTarget = false); 703 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) { 704 return getConstantFP(Val, DL, VT, true); 705 } 706 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) { 707 return getConstantFP(Val, DL, VT, true); 708 } 709 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) { 710 return getConstantFP(Val, DL, VT, true); 711 } 712 /// @} 713 714 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 715 int64_t offset = 0, bool isTargetGA = false, 716 unsigned TargetFlags = 0); 717 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 718 int64_t offset = 0, unsigned TargetFlags = 0) { 719 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); 720 } 721 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); 722 SDValue getTargetFrameIndex(int FI, EVT VT) { 723 return getFrameIndex(FI, VT, true); 724 } 725 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false, 726 unsigned TargetFlags = 0); 727 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) { 728 return getJumpTable(JTI, VT, true, TargetFlags); 729 } 730 SDValue getConstantPool(const Constant *C, EVT VT, 731 MaybeAlign Align = std::nullopt, int Offs = 0, 732 bool isT = false, unsigned TargetFlags = 0); 733 SDValue getTargetConstantPool(const Constant *C, EVT VT, 734 MaybeAlign Align = std::nullopt, int Offset = 0, 735 unsigned TargetFlags = 0) { 736 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 737 } 738 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT, 739 MaybeAlign Align = std::nullopt, int Offs = 0, 740 bool isT = false, unsigned TargetFlags = 0); 741 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, 742 MaybeAlign Align = std::nullopt, int Offset = 0, 743 unsigned TargetFlags = 0) { 744 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 745 } 746 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0, 747 unsigned TargetFlags = 0); 748 // When generating a branch to a BB, we don't in general know enough 749 // to provide debug info for the BB at that time, so keep this one around. 750 SDValue getBasicBlock(MachineBasicBlock *MBB); 751 SDValue getExternalSymbol(const char *Sym, EVT VT); 752 SDValue getTargetExternalSymbol(const char *Sym, EVT VT, 753 unsigned TargetFlags = 0); 754 SDValue getMCSymbol(MCSymbol *Sym, EVT VT); 755 756 SDValue getValueType(EVT); 757 SDValue getRegister(unsigned Reg, EVT VT); 758 SDValue getRegisterMask(const uint32_t *RegMask); 759 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); 760 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, 761 MCSymbol *Label); 762 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, 763 bool isTarget = false, unsigned TargetFlags = 0); 764 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, 765 int64_t Offset = 0, unsigned TargetFlags = 0) { 766 return getBlockAddress(BA, VT, Offset, true, TargetFlags); 767 } 768 769 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, 770 SDValue N) { 771 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain, 772 getRegister(Reg, N.getValueType()), N); 773 } 774 775 // This version of the getCopyToReg method takes an extra operand, which 776 // indicates that there is potentially an incoming glue value (if Glue is not 777 // null) and that there should be a glue result. 778 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N, 779 SDValue Glue) { 780 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 781 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue }; 782 return getNode(ISD::CopyToReg, dl, VTs, 783 ArrayRef(Ops, Glue.getNode() ? 4 : 3)); 784 } 785 786 // Similar to last getCopyToReg() except parameter Reg is a SDValue 787 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N, 788 SDValue Glue) { 789 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 790 SDValue Ops[] = { Chain, Reg, N, Glue }; 791 return getNode(ISD::CopyToReg, dl, VTs, 792 ArrayRef(Ops, Glue.getNode() ? 4 : 3)); 793 } 794 795 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) { 796 SDVTList VTs = getVTList(VT, MVT::Other); 797 SDValue Ops[] = { Chain, getRegister(Reg, VT) }; 798 return getNode(ISD::CopyFromReg, dl, VTs, Ops); 799 } 800 801 // This version of the getCopyFromReg method takes an extra operand, which 802 // indicates that there is potentially an incoming glue value (if Glue is not 803 // null) and that there should be a glue result. 804 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT, 805 SDValue Glue) { 806 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue); 807 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue }; 808 return getNode(ISD::CopyFromReg, dl, VTs, 809 ArrayRef(Ops, Glue.getNode() ? 3 : 2)); 810 } 811 812 SDValue getCondCode(ISD::CondCode Cond); 813 814 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT, 815 /// which must be a vector type, must match the number of mask elements 816 /// NumElts. An integer mask element equal to -1 is treated as undefined. 817 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, 818 ArrayRef<int> Mask); 819 820 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 821 /// which must be a vector type, must match the number of operands in Ops. 822 /// The operands must have the same type as (or, for integers, a type wider 823 /// than) VT's element type. 824 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) { 825 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 826 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 827 } 828 829 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 830 /// which must be a vector type, must match the number of operands in Ops. 831 /// The operands must have the same type as (or, for integers, a type wider 832 /// than) VT's element type. 833 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) { 834 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 835 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 836 } 837 838 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all 839 /// elements. VT must be a vector type. Op's type must be the same as (or, 840 /// for integers, a type wider than) VT's element type. 841 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) { 842 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 843 if (Op.getOpcode() == ISD::UNDEF) { 844 assert((VT.getVectorElementType() == Op.getValueType() || 845 (VT.isInteger() && 846 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 847 "A splatted value must have a width equal or (for integers) " 848 "greater than the vector element type!"); 849 return getNode(ISD::UNDEF, SDLoc(), VT); 850 } 851 852 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op); 853 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 854 } 855 856 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all 857 // elements. 858 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) { 859 if (Op.getOpcode() == ISD::UNDEF) { 860 assert((VT.getVectorElementType() == Op.getValueType() || 861 (VT.isInteger() && 862 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 863 "A splatted value must have a width equal or (for integers) " 864 "greater than the vector element type!"); 865 return getNode(ISD::UNDEF, SDLoc(), VT); 866 } 867 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op); 868 } 869 870 /// Returns a node representing a splat of one value into all lanes 871 /// of the provided vector type. This is a utility which returns 872 /// either a BUILD_VECTOR or SPLAT_VECTOR depending on the 873 /// scalability of the desired vector type. 874 SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op) { 875 assert(VT.isVector() && "Can't splat to non-vector type"); 876 return VT.isScalableVector() ? 877 getSplatVector(VT, DL, Op) : getSplatBuildVector(VT, DL, Op); 878 } 879 880 /// Returns a vector of type ResVT whose elements contain the linear sequence 881 /// <0, Step, Step * 2, Step * 3, ...> 882 SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); 883 884 /// Returns a vector of type ResVT whose elements contain the linear sequence 885 /// <0, 1, 2, 3, ...> 886 SDValue getStepVector(const SDLoc &DL, EVT ResVT); 887 888 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to 889 /// the shuffle node in input but with swapped operands. 890 /// 891 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3> 892 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV); 893 894 /// Convert Op, which must be of float type, to the 895 /// float type VT, by either extending or rounding (by truncation). 896 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT); 897 898 /// Convert Op, which must be a STRICT operation of float type, to the 899 /// float type VT, by either extending or rounding (by truncation). 900 std::pair<SDValue, SDValue> 901 getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT); 902 903 /// Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode. 904 static unsigned getOpcode_EXTEND(unsigned Opcode) { 905 switch (Opcode) { 906 case ISD::ANY_EXTEND: 907 case ISD::ANY_EXTEND_VECTOR_INREG: 908 return ISD::ANY_EXTEND; 909 case ISD::ZERO_EXTEND: 910 case ISD::ZERO_EXTEND_VECTOR_INREG: 911 return ISD::ZERO_EXTEND; 912 case ISD::SIGN_EXTEND: 913 case ISD::SIGN_EXTEND_VECTOR_INREG: 914 return ISD::SIGN_EXTEND; 915 } 916 llvm_unreachable("Unknown opcode"); 917 } 918 919 /// Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode. 920 static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) { 921 switch (Opcode) { 922 case ISD::ANY_EXTEND: 923 case ISD::ANY_EXTEND_VECTOR_INREG: 924 return ISD::ANY_EXTEND_VECTOR_INREG; 925 case ISD::ZERO_EXTEND: 926 case ISD::ZERO_EXTEND_VECTOR_INREG: 927 return ISD::ZERO_EXTEND_VECTOR_INREG; 928 case ISD::SIGN_EXTEND: 929 case ISD::SIGN_EXTEND_VECTOR_INREG: 930 return ISD::SIGN_EXTEND_VECTOR_INREG; 931 } 932 llvm_unreachable("Unknown opcode"); 933 } 934 935 /// Convert Op, which must be of integer type, to the 936 /// integer type VT, by either any-extending or truncating it. 937 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 938 939 /// Convert Op, which must be of integer type, to the 940 /// integer type VT, by either sign-extending or truncating it. 941 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 942 943 /// Convert Op, which must be of integer type, to the 944 /// integer type VT, by either zero-extending or truncating it. 945 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 946 947 /// Return the expression required to zero extend the Op 948 /// value assuming it was the smaller SrcTy value. 949 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 950 951 /// Convert Op, which must be of integer type, to the integer type VT, by 952 /// either truncating it or performing either zero or sign extension as 953 /// appropriate extension for the pointer's semantics. 954 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 955 956 /// Return the expression required to extend the Op as a pointer value 957 /// assuming it was the smaller SrcTy value. This may be either a zero extend 958 /// or a sign extend. 959 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 960 961 /// Convert Op, which must be of integer type, to the integer type VT, 962 /// by using an extension appropriate for the target's 963 /// BooleanContent for type OpVT or truncating it. 964 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT); 965 966 /// Create negative operation as (SUB 0, Val). 967 SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT); 968 969 /// Create a bitwise NOT operation as (XOR Val, -1). 970 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT); 971 972 /// Create a logical NOT operation as (XOR Val, BooleanOne). 973 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); 974 975 /// Create a vector-predicated logical NOT operation as (VP_XOR Val, 976 /// BooleanOne, Mask, EVL). 977 SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, 978 SDValue EVL, EVT VT); 979 980 /// Convert a vector-predicated Op, which must be an integer vector, to the 981 /// vector-type VT, by performing either vector-predicated zext or truncating 982 /// it. The Op will be returned as-is if Op and VT are vectors containing 983 /// integer with same width. 984 SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, 985 SDValue EVL); 986 987 /// Convert a vector-predicated Op, which must be of integer type, to the 988 /// vector-type integer type VT, by either truncating it or performing either 989 /// vector-predicated zero or sign extension as appropriate extension for the 990 /// pointer's semantics. This function just redirects to getVPZExtOrTrunc 991 /// right now. 992 SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, 993 SDValue EVL); 994 995 /// Returns sum of the base pointer and offset. 996 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. 997 SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, 998 const SDNodeFlags Flags = SDNodeFlags()); 999 SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL, 1000 const SDNodeFlags Flags = SDNodeFlags()); 1001 1002 /// Create an add instruction with appropriate flags when used for 1003 /// addressing some offset of an object. i.e. if a load is split into multiple 1004 /// components, create an add nuw from the base pointer to the offset. 1005 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) { 1006 SDNodeFlags Flags; 1007 Flags.setNoUnsignedWrap(true); 1008 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 1009 } 1010 1011 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { 1012 // The object itself can't wrap around the address space, so it shouldn't be 1013 // possible for the adds of the offsets to the split parts to overflow. 1014 SDNodeFlags Flags; 1015 Flags.setNoUnsignedWrap(true); 1016 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 1017 } 1018 1019 /// Return a new CALLSEQ_START node, that starts new call frame, in which 1020 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and 1021 /// OutSize specifies part of the frame set up prior to the sequence. 1022 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, 1023 const SDLoc &DL) { 1024 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 1025 SDValue Ops[] = { Chain, 1026 getIntPtrConstant(InSize, DL, true), 1027 getIntPtrConstant(OutSize, DL, true) }; 1028 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops); 1029 } 1030 1031 /// Return a new CALLSEQ_END node, which always must have a 1032 /// glue result (to ensure it's not CSE'd). 1033 /// CALLSEQ_END does not have a useful SDLoc. 1034 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, 1035 SDValue InGlue, const SDLoc &DL) { 1036 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue); 1037 SmallVector<SDValue, 4> Ops; 1038 Ops.push_back(Chain); 1039 Ops.push_back(Op1); 1040 Ops.push_back(Op2); 1041 if (InGlue.getNode()) 1042 Ops.push_back(InGlue); 1043 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops); 1044 } 1045 1046 SDValue getCALLSEQ_END(SDValue Chain, uint64_t Size1, uint64_t Size2, 1047 SDValue Glue, const SDLoc &DL) { 1048 return getCALLSEQ_END( 1049 Chain, getIntPtrConstant(Size1, DL, /*isTarget=*/true), 1050 getIntPtrConstant(Size2, DL, /*isTarget=*/true), Glue, DL); 1051 } 1052 1053 /// Return true if the result of this operation is always undefined. 1054 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops); 1055 1056 /// Return an UNDEF node. UNDEF does not have a useful SDLoc. 1057 SDValue getUNDEF(EVT VT) { 1058 return getNode(ISD::UNDEF, SDLoc(), VT); 1059 } 1060 1061 /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'. 1062 SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) { 1063 assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() && 1064 "Immediate does not fit VT"); 1065 return getNode(ISD::VSCALE, DL, VT, 1066 getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT)); 1067 } 1068 1069 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc. 1070 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) { 1071 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); 1072 } 1073 1074 /// Gets or creates the specified node. 1075 /// 1076 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1077 ArrayRef<SDUse> Ops); 1078 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1079 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1080 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, 1081 ArrayRef<SDValue> Ops); 1082 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 1083 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1084 1085 // Use flags from current flag inserter. 1086 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1087 ArrayRef<SDValue> Ops); 1088 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 1089 ArrayRef<SDValue> Ops); 1090 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand); 1091 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1092 SDValue N2); 1093 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1094 SDValue N2, SDValue N3); 1095 1096 // Specialize based on number of operands. 1097 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT); 1098 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, 1099 const SDNodeFlags Flags); 1100 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1101 SDValue N2, const SDNodeFlags Flags); 1102 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1103 SDValue N2, SDValue N3, const SDNodeFlags Flags); 1104 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1105 SDValue N2, SDValue N3, SDValue N4); 1106 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1107 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 1108 1109 // Specialize again based on number of operands for nodes with a VTList 1110 // rather than a single VT. 1111 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList); 1112 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N); 1113 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1114 SDValue N2); 1115 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1116 SDValue N2, SDValue N3); 1117 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1118 SDValue N2, SDValue N3, SDValue N4); 1119 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1120 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 1121 1122 /// Compute a TokenFactor to force all the incoming stack arguments to be 1123 /// loaded from the stack. This is used in tail call lowering to protect 1124 /// stack arguments from being clobbered. 1125 SDValue getStackArgumentTokenFactor(SDValue Chain); 1126 1127 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1128 SDValue Size, Align Alignment, bool isVol, 1129 bool AlwaysInline, bool isTailCall, 1130 MachinePointerInfo DstPtrInfo, 1131 MachinePointerInfo SrcPtrInfo, 1132 const AAMDNodes &AAInfo = AAMDNodes(), 1133 AAResults *AA = nullptr); 1134 1135 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1136 SDValue Size, Align Alignment, bool isVol, bool isTailCall, 1137 MachinePointerInfo DstPtrInfo, 1138 MachinePointerInfo SrcPtrInfo, 1139 const AAMDNodes &AAInfo = AAMDNodes(), 1140 AAResults *AA = nullptr); 1141 1142 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1143 SDValue Size, Align Alignment, bool isVol, 1144 bool AlwaysInline, bool isTailCall, 1145 MachinePointerInfo DstPtrInfo, 1146 const AAMDNodes &AAInfo = AAMDNodes()); 1147 1148 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 1149 SDValue Src, SDValue Size, Type *SizeTy, 1150 unsigned ElemSz, bool isTailCall, 1151 MachinePointerInfo DstPtrInfo, 1152 MachinePointerInfo SrcPtrInfo); 1153 1154 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 1155 SDValue Src, SDValue Size, Type *SizeTy, 1156 unsigned ElemSz, bool isTailCall, 1157 MachinePointerInfo DstPtrInfo, 1158 MachinePointerInfo SrcPtrInfo); 1159 1160 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 1161 SDValue Value, SDValue Size, Type *SizeTy, 1162 unsigned ElemSz, bool isTailCall, 1163 MachinePointerInfo DstPtrInfo); 1164 1165 /// Helper function to make it easier to build SetCC's if you just have an 1166 /// ISD::CondCode instead of an SDValue. 1167 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, 1168 ISD::CondCode Cond, SDValue Chain = SDValue(), 1169 bool IsSignaling = false) { 1170 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && 1171 "Vector/scalar operand type mismatch for setcc"); 1172 assert(LHS.getValueType().isVector() == VT.isVector() && 1173 "Vector/scalar result type mismatch for setcc"); 1174 assert(Cond != ISD::SETCC_INVALID && 1175 "Cannot create a setCC of an invalid node."); 1176 if (Chain) 1177 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL, 1178 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)}); 1179 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); 1180 } 1181 1182 /// Helper function to make it easier to build VP_SETCCs if you just have an 1183 /// ISD::CondCode instead of an SDValue. 1184 SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, 1185 ISD::CondCode Cond, SDValue Mask, SDValue EVL) { 1186 assert(LHS.getValueType().isVector() && RHS.getValueType().isVector() && 1187 "Cannot compare scalars"); 1188 assert(Cond != ISD::SETCC_INVALID && 1189 "Cannot create a setCC of an invalid node."); 1190 return getNode(ISD::VP_SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Mask, 1191 EVL); 1192 } 1193 1194 /// Helper function to make it easier to build Select's if you just have 1195 /// operands and don't want to check for vector. 1196 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, 1197 SDValue RHS) { 1198 assert(LHS.getValueType() == VT && RHS.getValueType() == VT && 1199 "Cannot use select on differing types"); 1200 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; 1201 return getNode(Opcode, DL, VT, Cond, LHS, RHS); 1202 } 1203 1204 /// Helper function to make it easier to build SelectCC's if you just have an 1205 /// ISD::CondCode instead of an SDValue. 1206 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, 1207 SDValue False, ISD::CondCode Cond) { 1208 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, 1209 False, getCondCode(Cond)); 1210 } 1211 1212 /// Try to simplify a select/vselect into 1 of its operands or a constant. 1213 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); 1214 1215 /// Try to simplify a shift into 1 of its operands or a constant. 1216 SDValue simplifyShift(SDValue X, SDValue Y); 1217 1218 /// Try to simplify a floating-point binary operation into 1 of its operands 1219 /// or a constant. 1220 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, 1221 SDNodeFlags Flags); 1222 1223 /// VAArg produces a result and token chain, and takes a pointer 1224 /// and a source value as input. 1225 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1226 SDValue SV, unsigned Align); 1227 1228 /// Gets a node for an atomic cmpxchg op. There are two 1229 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a 1230 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, 1231 /// a success flag (initially i1), and a chain. 1232 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1233 SDVTList VTs, SDValue Chain, SDValue Ptr, 1234 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); 1235 1236 /// Gets a node for an atomic op, produces result (if relevant) 1237 /// and chain and takes 2 operands. 1238 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, 1239 SDValue Ptr, SDValue Val, MachineMemOperand *MMO); 1240 1241 /// Gets a node for an atomic op, produces result and chain and 1242 /// takes 1 operand. 1243 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, 1244 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); 1245 1246 /// Gets a node for an atomic op, produces result and chain and takes N 1247 /// operands. 1248 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1249 SDVTList VTList, ArrayRef<SDValue> Ops, 1250 MachineMemOperand *MMO); 1251 1252 /// Creates a MemIntrinsicNode that may produce a 1253 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, 1254 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not 1255 /// less than FIRST_TARGET_MEMORY_OPCODE. 1256 SDValue getMemIntrinsicNode( 1257 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1258 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, 1259 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1260 MachineMemOperand::MOStore, 1261 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()); 1262 1263 inline SDValue getMemIntrinsicNode( 1264 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1265 EVT MemVT, MachinePointerInfo PtrInfo, 1266 MaybeAlign Alignment = std::nullopt, 1267 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1268 MachineMemOperand::MOStore, 1269 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) { 1270 // Ensure that codegen never sees alignment 0 1271 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, 1272 Alignment.value_or(getEVTAlign(MemVT)), Flags, 1273 Size, AAInfo); 1274 } 1275 1276 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, 1277 ArrayRef<SDValue> Ops, EVT MemVT, 1278 MachineMemOperand *MMO); 1279 1280 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends 1281 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between 1282 /// offsets `Offset` and `Offset + Size`. 1283 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, 1284 int FrameIndex, int64_t Size, int64_t Offset = -1); 1285 1286 /// Creates a PseudoProbeSDNode with function GUID `Guid` and 1287 /// the index of the block `Index` it is probing, as well as the attributes 1288 /// `attr` of the probe. 1289 SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, 1290 uint64_t Index, uint32_t Attr); 1291 1292 /// Create a MERGE_VALUES node from the given operands. 1293 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl); 1294 1295 /// Loads are not normal binary operators: their result type is not 1296 /// determined by their operands, and they produce a value AND a token chain. 1297 /// 1298 /// This function will set the MOLoad flag on MMOFlags, but you can set it if 1299 /// you want. The MOStore flag must not be set. 1300 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1301 MachinePointerInfo PtrInfo, 1302 MaybeAlign Alignment = MaybeAlign(), 1303 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1304 const AAMDNodes &AAInfo = AAMDNodes(), 1305 const MDNode *Ranges = nullptr); 1306 /// FIXME: Remove once transition to Align is over. 1307 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", 1308 "") 1309 inline SDValue 1310 getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1311 MachinePointerInfo PtrInfo, unsigned Alignment, 1312 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1313 const AAMDNodes &AAInfo = AAMDNodes(), 1314 const MDNode *Ranges = nullptr) { 1315 return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags, 1316 AAInfo, Ranges); 1317 } 1318 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1319 MachineMemOperand *MMO); 1320 SDValue 1321 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, 1322 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, 1323 MaybeAlign Alignment = MaybeAlign(), 1324 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1325 const AAMDNodes &AAInfo = AAMDNodes()); 1326 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1327 SDValue Chain, SDValue Ptr, EVT MemVT, 1328 MachineMemOperand *MMO); 1329 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1330 SDValue Offset, ISD::MemIndexedMode AM); 1331 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1332 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1333 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, 1334 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1335 const AAMDNodes &AAInfo = AAMDNodes(), 1336 const MDNode *Ranges = nullptr); 1337 inline SDValue getLoad( 1338 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, 1339 SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, 1340 EVT MemVT, MaybeAlign Alignment = MaybeAlign(), 1341 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1342 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) { 1343 // Ensures that codegen never sees a None Alignment. 1344 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1345 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo, 1346 Ranges); 1347 } 1348 /// FIXME: Remove once transition to Align is over. 1349 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", 1350 "") 1351 inline SDValue 1352 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1353 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1354 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment, 1355 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1356 const AAMDNodes &AAInfo = AAMDNodes(), 1357 const MDNode *Ranges = nullptr) { 1358 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1359 MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges); 1360 } 1361 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1362 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1363 EVT MemVT, MachineMemOperand *MMO); 1364 1365 /// Helper function to build ISD::STORE nodes. 1366 /// 1367 /// This function will set the MOStore flag on MMOFlags, but you can set it if 1368 /// you want. The MOLoad and MOInvariant flags must not be set. 1369 1370 SDValue 1371 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1372 MachinePointerInfo PtrInfo, Align Alignment, 1373 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1374 const AAMDNodes &AAInfo = AAMDNodes()); 1375 inline SDValue 1376 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1377 MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(), 1378 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1379 const AAMDNodes &AAInfo = AAMDNodes()) { 1380 return getStore(Chain, dl, Val, Ptr, PtrInfo, 1381 Alignment.value_or(getEVTAlign(Val.getValueType())), 1382 MMOFlags, AAInfo); 1383 } 1384 /// FIXME: Remove once transition to Align is over. 1385 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "") 1386 inline SDValue 1387 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1388 MachinePointerInfo PtrInfo, unsigned Alignment, 1389 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1390 const AAMDNodes &AAInfo = AAMDNodes()) { 1391 return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment), 1392 MMOFlags, AAInfo); 1393 } 1394 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1395 MachineMemOperand *MMO); 1396 SDValue 1397 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1398 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, 1399 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1400 const AAMDNodes &AAInfo = AAMDNodes()); 1401 inline SDValue 1402 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1403 MachinePointerInfo PtrInfo, EVT SVT, 1404 MaybeAlign Alignment = MaybeAlign(), 1405 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1406 const AAMDNodes &AAInfo = AAMDNodes()) { 1407 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1408 Alignment.value_or(getEVTAlign(SVT)), MMOFlags, 1409 AAInfo); 1410 } 1411 /// FIXME: Remove once transition to Align is over. 1412 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "") 1413 inline SDValue 1414 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1415 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment, 1416 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1417 const AAMDNodes &AAInfo = AAMDNodes()) { 1418 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1419 MaybeAlign(Alignment), MMOFlags, AAInfo); 1420 } 1421 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1422 SDValue Ptr, EVT SVT, MachineMemOperand *MMO); 1423 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, 1424 SDValue Offset, ISD::MemIndexedMode AM); 1425 1426 SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1427 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1428 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, 1429 EVT MemVT, Align Alignment, 1430 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, 1431 const MDNode *Ranges = nullptr, bool IsExpanding = false); 1432 inline SDValue 1433 getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1434 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1435 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1436 MaybeAlign Alignment = MaybeAlign(), 1437 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1438 const AAMDNodes &AAInfo = AAMDNodes(), 1439 const MDNode *Ranges = nullptr, bool IsExpanding = false) { 1440 // Ensures that codegen never sees a None Alignment. 1441 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, 1442 PtrInfo, MemVT, Alignment.value_or(getEVTAlign(MemVT)), 1443 MMOFlags, AAInfo, Ranges, IsExpanding); 1444 } 1445 SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1446 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1447 SDValue Mask, SDValue EVL, EVT MemVT, 1448 MachineMemOperand *MMO, bool IsExpanding = false); 1449 SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1450 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, 1451 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1452 const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr, 1453 bool IsExpanding = false); 1454 SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1455 SDValue Mask, SDValue EVL, MachineMemOperand *MMO, 1456 bool IsExpanding = false); 1457 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1458 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, 1459 MachinePointerInfo PtrInfo, EVT MemVT, 1460 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1461 const AAMDNodes &AAInfo, bool IsExpanding = false); 1462 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1463 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, 1464 EVT MemVT, MachineMemOperand *MMO, 1465 bool IsExpanding = false); 1466 SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1467 SDValue Offset, ISD::MemIndexedMode AM); 1468 SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1469 SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, 1470 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1471 bool IsTruncating = false, bool IsCompressing = false); 1472 SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, 1473 SDValue Ptr, SDValue Mask, SDValue EVL, 1474 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, 1475 MachineMemOperand::Flags MMOFlags, 1476 const AAMDNodes &AAInfo, bool IsCompressing = false); 1477 SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, 1478 SDValue Ptr, SDValue Mask, SDValue EVL, EVT SVT, 1479 MachineMemOperand *MMO, bool IsCompressing = false); 1480 SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, 1481 SDValue Offset, ISD::MemIndexedMode AM); 1482 1483 SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 1484 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1485 SDValue Offset, SDValue Stride, SDValue Mask, 1486 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1487 Align Alignment, MachineMemOperand::Flags MMOFlags, 1488 const AAMDNodes &AAInfo, 1489 const MDNode *Ranges = nullptr, 1490 bool IsExpanding = false); 1491 inline SDValue getStridedLoadVP( 1492 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, 1493 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, 1494 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1495 MaybeAlign Alignment = MaybeAlign(), 1496 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1497 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, 1498 bool IsExpanding = false) { 1499 // Ensures that codegen never sees a None Alignment. 1500 return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, 1501 Mask, EVL, PtrInfo, MemVT, 1502 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, 1503 AAInfo, Ranges, IsExpanding); 1504 } 1505 SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 1506 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1507 SDValue Offset, SDValue Stride, SDValue Mask, 1508 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, 1509 bool IsExpanding = false); 1510 SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1511 SDValue Stride, SDValue Mask, SDValue EVL, 1512 MachinePointerInfo PtrInfo, MaybeAlign Alignment, 1513 MachineMemOperand::Flags MMOFlags, 1514 const AAMDNodes &AAInfo, 1515 const MDNode *Ranges = nullptr, 1516 bool IsExpanding = false); 1517 SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1518 SDValue Stride, SDValue Mask, SDValue EVL, 1519 MachineMemOperand *MMO, bool IsExpanding = false); 1520 SDValue 1521 getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, 1522 SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, 1523 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1524 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1525 const AAMDNodes &AAInfo, bool IsExpanding = false); 1526 SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, 1527 SDValue Chain, SDValue Ptr, SDValue Stride, 1528 SDValue Mask, SDValue EVL, EVT MemVT, 1529 MachineMemOperand *MMO, bool IsExpanding = false); 1530 SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, 1531 SDValue Base, SDValue Offset, 1532 ISD::MemIndexedMode AM); 1533 SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1534 SDValue Ptr, SDValue Offset, SDValue Stride, 1535 SDValue Mask, SDValue EVL, EVT MemVT, 1536 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1537 bool IsTruncating = false, 1538 bool IsCompressing = false); 1539 SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1540 SDValue Ptr, SDValue Stride, SDValue Mask, 1541 SDValue EVL, MachinePointerInfo PtrInfo, 1542 EVT SVT, Align Alignment, 1543 MachineMemOperand::Flags MMOFlags, 1544 const AAMDNodes &AAInfo, 1545 bool IsCompressing = false); 1546 SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1547 SDValue Ptr, SDValue Stride, SDValue Mask, 1548 SDValue EVL, EVT SVT, MachineMemOperand *MMO, 1549 bool IsCompressing = false); 1550 SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, 1551 SDValue Base, SDValue Offset, 1552 ISD::MemIndexedMode AM); 1553 1554 SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, 1555 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1556 ISD::MemIndexType IndexType); 1557 SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, 1558 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1559 ISD::MemIndexType IndexType); 1560 1561 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, 1562 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, 1563 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1564 ISD::LoadExtType, bool IsExpanding = false); 1565 SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1566 SDValue Offset, ISD::MemIndexedMode AM); 1567 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1568 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, 1569 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1570 bool IsTruncating = false, bool IsCompressing = false); 1571 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, 1572 SDValue Base, SDValue Offset, 1573 ISD::MemIndexedMode AM); 1574 SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, 1575 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1576 ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy); 1577 SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, 1578 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1579 ISD::MemIndexType IndexType, 1580 bool IsTruncating = false); 1581 1582 /// Construct a node to track a Value* through the backend. 1583 SDValue getSrcValue(const Value *v); 1584 1585 /// Return an MDNodeSDNode which holds an MDNode. 1586 SDValue getMDNode(const MDNode *MD); 1587 1588 /// Return a bitcast using the SDLoc of the value operand, and casting to the 1589 /// provided type. Use getNode to set a custom SDLoc. 1590 SDValue getBitcast(EVT VT, SDValue V); 1591 1592 /// Return an AddrSpaceCastSDNode. 1593 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, 1594 unsigned DestAS); 1595 1596 /// Return a freeze using the SDLoc of the value operand. 1597 SDValue getFreeze(SDValue V); 1598 1599 /// Return an AssertAlignSDNode. 1600 SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A); 1601 1602 /// Swap N1 and N2 if Opcode is a commutative binary opcode 1603 /// and the canonical form expects the opposite order. 1604 void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, 1605 SDValue &N2) const; 1606 1607 /// Return the specified value casted to 1608 /// the target's desired shift amount type. 1609 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); 1610 1611 /// Expand the specified \c ISD::VAARG node as the Legalize pass would. 1612 SDValue expandVAArg(SDNode *Node); 1613 1614 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. 1615 SDValue expandVACopy(SDNode *Node); 1616 1617 /// Returs an GlobalAddress of the function from the current module with 1618 /// name matching the given ExternalSymbol. Additionally can provide the 1619 /// matched function. 1620 /// Panics the function doesn't exists. 1621 SDValue getSymbolFunctionGlobalAddress(SDValue Op, 1622 Function **TargetFunction = nullptr); 1623 1624 /// *Mutate* the specified node in-place to have the 1625 /// specified operands. If the resultant node already exists in the DAG, 1626 /// this does not modify the specified node, instead it returns the node that 1627 /// already exists. If the resultant node does not exist in the DAG, the 1628 /// input node is returned. As a degenerate case, if you specify the same 1629 /// input operands as the node already has, the input node is returned. 1630 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op); 1631 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2); 1632 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1633 SDValue Op3); 1634 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1635 SDValue Op3, SDValue Op4); 1636 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1637 SDValue Op3, SDValue Op4, SDValue Op5); 1638 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); 1639 1640 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k 1641 /// values or more, move values into new TokenFactors in 64k-1 blocks, until 1642 /// the final TokenFactor has less than 64k operands. 1643 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals); 1644 1645 /// *Mutate* the specified machine node's memory references to the provided 1646 /// list. 1647 void setNodeMemRefs(MachineSDNode *N, 1648 ArrayRef<MachineMemOperand *> NewMemRefs); 1649 1650 // Calculate divergence of node \p N based on its operands. 1651 bool calculateDivergence(SDNode *N); 1652 1653 // Propagates the change in divergence to users 1654 void updateDivergence(SDNode * N); 1655 1656 /// These are used for target selectors to *mutate* the 1657 /// specified node to have the specified return type, Target opcode, and 1658 /// operands. Note that target opcodes are stored as 1659 /// ~TargetOpcode in the node opcode field. The resultant node is returned. 1660 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT); 1661 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1); 1662 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1663 SDValue Op1, SDValue Op2); 1664 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1665 SDValue Op1, SDValue Op2, SDValue Op3); 1666 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1667 ArrayRef<SDValue> Ops); 1668 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2); 1669 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1670 EVT VT2, ArrayRef<SDValue> Ops); 1671 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1672 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1673 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1674 EVT VT2, SDValue Op1, SDValue Op2); 1675 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs, 1676 ArrayRef<SDValue> Ops); 1677 1678 /// This *mutates* the specified node to have the specified 1679 /// return type, opcode, and operands. 1680 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, 1681 ArrayRef<SDValue> Ops); 1682 1683 /// Mutate the specified strict FP node to its non-strict equivalent, 1684 /// unlinking the node from its chain and dropping the metadata arguments. 1685 /// The node must be a strict FP node. 1686 SDNode *mutateStrictFPToFP(SDNode *Node); 1687 1688 /// These are used for target selectors to create a new node 1689 /// with specified return type(s), MachineInstr opcode, and operands. 1690 /// 1691 /// Note that getMachineNode returns the resultant node. If there is already 1692 /// a node of the specified opcode and operands, it returns that node instead 1693 /// of the current one. 1694 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT); 1695 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1696 SDValue Op1); 1697 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1698 SDValue Op1, SDValue Op2); 1699 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1700 SDValue Op1, SDValue Op2, SDValue Op3); 1701 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1702 ArrayRef<SDValue> Ops); 1703 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1704 EVT VT2, SDValue Op1, SDValue Op2); 1705 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1706 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3); 1707 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1708 EVT VT2, ArrayRef<SDValue> Ops); 1709 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1710 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2); 1711 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1712 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2, 1713 SDValue Op3); 1714 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1715 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1716 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, 1717 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); 1718 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs, 1719 ArrayRef<SDValue> Ops); 1720 1721 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes. 1722 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1723 SDValue Operand); 1724 1725 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes. 1726 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1727 SDValue Operand, SDValue Subreg); 1728 1729 /// Get the specified node if it's already available, or else return NULL. 1730 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, 1731 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1732 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, 1733 ArrayRef<SDValue> Ops); 1734 1735 /// Check if a node exists without modifying its flags. 1736 bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops); 1737 1738 /// Creates a SDDbgValue node. 1739 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, 1740 unsigned R, bool IsIndirect, const DebugLoc &DL, 1741 unsigned O); 1742 1743 /// Creates a constant SDDbgValue node. 1744 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr, 1745 const Value *C, const DebugLoc &DL, 1746 unsigned O); 1747 1748 /// Creates a FrameIndex SDDbgValue node. 1749 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, 1750 unsigned FI, bool IsIndirect, 1751 const DebugLoc &DL, unsigned O); 1752 1753 /// Creates a FrameIndex SDDbgValue node. 1754 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, 1755 unsigned FI, 1756 ArrayRef<SDNode *> Dependencies, 1757 bool IsIndirect, const DebugLoc &DL, 1758 unsigned O); 1759 1760 /// Creates a VReg SDDbgValue node. 1761 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr, 1762 unsigned VReg, bool IsIndirect, 1763 const DebugLoc &DL, unsigned O); 1764 1765 /// Creates a SDDbgValue node from a list of locations. 1766 SDDbgValue *getDbgValueList(DIVariable *Var, DIExpression *Expr, 1767 ArrayRef<SDDbgOperand> Locs, 1768 ArrayRef<SDNode *> Dependencies, bool IsIndirect, 1769 const DebugLoc &DL, unsigned O, bool IsVariadic); 1770 1771 /// Creates a SDDbgLabel node. 1772 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O); 1773 1774 /// Transfer debug values from one node to another, while optionally 1775 /// generating fragment expressions for split-up values. If \p InvalidateDbg 1776 /// is set, debug values are invalidated after they are transferred. 1777 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0, 1778 unsigned SizeInBits = 0, bool InvalidateDbg = true); 1779 1780 /// Remove the specified node from the system. If any of its 1781 /// operands then becomes dead, remove them as well. Inform UpdateListener 1782 /// for each node deleted. 1783 void RemoveDeadNode(SDNode *N); 1784 1785 /// This method deletes the unreachable nodes in the 1786 /// given list, and any nodes that become unreachable as a result. 1787 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes); 1788 1789 /// Modify anything using 'From' to use 'To' instead. 1790 /// This can cause recursive merging of nodes in the DAG. Use the first 1791 /// version if 'From' is known to have a single result, use the second 1792 /// if you have two nodes with identical results (or if 'To' has a superset 1793 /// of the results of 'From'), use the third otherwise. 1794 /// 1795 /// These methods all take an optional UpdateListener, which (if not null) is 1796 /// informed about nodes that are deleted and modified due to recursive 1797 /// changes in the dag. 1798 /// 1799 /// These functions only replace all existing uses. It's possible that as 1800 /// these replacements are being performed, CSE may cause the From node 1801 /// to be given new uses. These new uses of From are left in place, and 1802 /// not automatically transferred to To. 1803 /// 1804 void ReplaceAllUsesWith(SDValue From, SDValue To); 1805 void ReplaceAllUsesWith(SDNode *From, SDNode *To); 1806 void ReplaceAllUsesWith(SDNode *From, const SDValue *To); 1807 1808 /// Replace any uses of From with To, leaving 1809 /// uses of other values produced by From.getNode() alone. 1810 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); 1811 1812 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. 1813 /// This correctly handles the case where 1814 /// there is an overlap between the From values and the To values. 1815 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, 1816 unsigned Num); 1817 1818 /// If an existing load has uses of its chain, create a token factor node with 1819 /// that chain and the new memory node's chain and update users of the old 1820 /// chain to the token factor. This ensures that the new memory node will have 1821 /// the same relative memory dependency position as the old load. Returns the 1822 /// new merged load chain. 1823 SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain); 1824 1825 /// If an existing load has uses of its chain, create a token factor node with 1826 /// that chain and the new memory node's chain and update users of the old 1827 /// chain to the token factor. This ensures that the new memory node will have 1828 /// the same relative memory dependency position as the old load. Returns the 1829 /// new merged load chain. 1830 SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp); 1831 1832 /// Topological-sort the AllNodes list and a 1833 /// assign a unique node id for each node in the DAG based on their 1834 /// topological order. Returns the number of nodes. 1835 unsigned AssignTopologicalOrder(); 1836 1837 /// Move node N in the AllNodes list to be immediately 1838 /// before the given iterator Position. This may be used to update the 1839 /// topological ordering when the list of nodes is modified. 1840 void RepositionNode(allnodes_iterator Position, SDNode *N) { 1841 AllNodes.insert(Position, AllNodes.remove(N)); 1842 } 1843 1844 /// Returns an APFloat semantics tag appropriate for the given type. If VT is 1845 /// a vector type, the element semantics are returned. 1846 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) { 1847 switch (VT.getScalarType().getSimpleVT().SimpleTy) { 1848 default: llvm_unreachable("Unknown FP format"); 1849 case MVT::f16: return APFloat::IEEEhalf(); 1850 case MVT::bf16: return APFloat::BFloat(); 1851 case MVT::f32: return APFloat::IEEEsingle(); 1852 case MVT::f64: return APFloat::IEEEdouble(); 1853 case MVT::f80: return APFloat::x87DoubleExtended(); 1854 case MVT::f128: return APFloat::IEEEquad(); 1855 case MVT::ppcf128: return APFloat::PPCDoubleDouble(); 1856 } 1857 } 1858 1859 /// Add a dbg_value SDNode. If SD is non-null that means the 1860 /// value is produced by SD. 1861 void AddDbgValue(SDDbgValue *DB, bool isParameter); 1862 1863 /// Add a dbg_label SDNode. 1864 void AddDbgLabel(SDDbgLabel *DB); 1865 1866 /// Get the debug values which reference the given SDNode. 1867 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const { 1868 return DbgInfo->getSDDbgValues(SD); 1869 } 1870 1871 public: 1872 /// Return true if there are any SDDbgValue nodes associated 1873 /// with this SelectionDAG. 1874 bool hasDebugValues() const { return !DbgInfo->empty(); } 1875 1876 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); } 1877 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); } 1878 1879 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const { 1880 return DbgInfo->ByvalParmDbgBegin(); 1881 } 1882 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const { 1883 return DbgInfo->ByvalParmDbgEnd(); 1884 } 1885 1886 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const { 1887 return DbgInfo->DbgLabelBegin(); 1888 } 1889 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const { 1890 return DbgInfo->DbgLabelEnd(); 1891 } 1892 1893 /// To be invoked on an SDNode that is slated to be erased. This 1894 /// function mirrors \c llvm::salvageDebugInfo. 1895 void salvageDebugInfo(SDNode &N); 1896 1897 void dump() const; 1898 1899 /// In most cases this function returns the ABI alignment for a given type, 1900 /// except for illegal vector types where the alignment exceeds that of the 1901 /// stack. In such cases we attempt to break the vector down to a legal type 1902 /// and return the ABI alignment for that instead. 1903 Align getReducedAlign(EVT VT, bool UseABI); 1904 1905 /// Create a stack temporary based on the size in bytes and the alignment 1906 SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment); 1907 1908 /// Create a stack temporary, suitable for holding the specified value type. 1909 /// If minAlign is specified, the slot size will have at least that alignment. 1910 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); 1911 1912 /// Create a stack temporary suitable for holding either of the specified 1913 /// value types. 1914 SDValue CreateStackTemporary(EVT VT1, EVT VT2); 1915 1916 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, 1917 const GlobalAddressSDNode *GA, 1918 const SDNode *N2); 1919 1920 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, 1921 ArrayRef<SDValue> Ops); 1922 1923 /// Fold floating-point operations with 2 operands when both operands are 1924 /// constants and/or undefined. 1925 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, 1926 SDValue N1, SDValue N2); 1927 1928 /// Constant fold a setcc to true or false. 1929 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, 1930 const SDLoc &dl); 1931 1932 /// Return true if the sign bit of Op is known to be zero. 1933 /// We use this predicate to simplify operations downstream. 1934 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; 1935 1936 /// Return true if 'Op & Mask' is known to be zero. We 1937 /// use this predicate to simplify operations downstream. Op and Mask are 1938 /// known to be the same type. 1939 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1940 unsigned Depth = 0) const; 1941 1942 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We 1943 /// use this predicate to simplify operations downstream. Op and Mask are 1944 /// known to be the same type. 1945 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1946 const APInt &DemandedElts, unsigned Depth = 0) const; 1947 1948 /// Return true if 'Op' is known to be zero in DemandedElts. We 1949 /// use this predicate to simplify operations downstream. 1950 bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, 1951 unsigned Depth = 0) const; 1952 1953 /// Return true if '(Op & Mask) == Mask'. 1954 /// Op and Mask are known to be the same type. 1955 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, 1956 unsigned Depth = 0) const; 1957 1958 /// For each demanded element of a vector, see if it is known to be zero. 1959 APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, 1960 unsigned Depth = 0) const; 1961 1962 /// Determine which bits of Op are known to be either zero or one and return 1963 /// them in Known. For vectors, the known bits are those that are shared by 1964 /// every vector element. 1965 /// Targets can implement the computeKnownBitsForTargetNode method in the 1966 /// TargetLowering class to allow target nodes to be understood. 1967 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; 1968 1969 /// Determine which bits of Op are known to be either zero or one and return 1970 /// them in Known. The DemandedElts argument allows us to only collect the 1971 /// known bits that are shared by the requested vector elements. 1972 /// Targets can implement the computeKnownBitsForTargetNode method in the 1973 /// TargetLowering class to allow target nodes to be understood. 1974 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, 1975 unsigned Depth = 0) const; 1976 1977 /// Used to represent the possible overflow behavior of an operation. 1978 /// Never: the operation cannot overflow. 1979 /// Always: the operation will always overflow. 1980 /// Sometime: the operation may or may not overflow. 1981 enum OverflowKind { 1982 OFK_Never, 1983 OFK_Sometime, 1984 OFK_Always, 1985 }; 1986 1987 /// Determine if the result of the addition of 2 node can overflow. 1988 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const; 1989 1990 /// Test if the given value is known to have exactly one bit set. This differs 1991 /// from computeKnownBits in that it doesn't necessarily determine which bit 1992 /// is set. 1993 bool isKnownToBeAPowerOfTwo(SDValue Val) const; 1994 1995 /// Return the number of times the sign bit of the register is replicated into 1996 /// the other bits. We know that at least 1 bit is always equal to the sign 1997 /// bit (itself), but other cases can give us information. For example, 1998 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 1999 /// to each other, so we return 3. Targets can implement the 2000 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow 2001 /// target nodes to be understood. 2002 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const; 2003 2004 /// Return the number of times the sign bit of the register is replicated into 2005 /// the other bits. We know that at least 1 bit is always equal to the sign 2006 /// bit (itself), but other cases can give us information. For example, 2007 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 2008 /// to each other, so we return 3. The DemandedElts argument allows 2009 /// us to only collect the minimum sign bits of the requested vector elements. 2010 /// Targets can implement the ComputeNumSignBitsForTarget method in the 2011 /// TargetLowering class to allow target nodes to be understood. 2012 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 2013 unsigned Depth = 0) const; 2014 2015 /// Get the upper bound on bit size for this Value \p Op as a signed integer. 2016 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). 2017 /// Similar to the APInt::getSignificantBits function. 2018 /// Helper wrapper to ComputeNumSignBits. 2019 unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth = 0) const; 2020 2021 /// Get the upper bound on bit size for this Value \p Op as a signed integer. 2022 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). 2023 /// Similar to the APInt::getSignificantBits function. 2024 /// Helper wrapper to ComputeNumSignBits. 2025 unsigned ComputeMaxSignificantBits(SDValue Op, const APInt &DemandedElts, 2026 unsigned Depth = 0) const; 2027 2028 /// Return true if this function can prove that \p Op is never poison 2029 /// and, if \p PoisonOnly is false, does not have undef bits. 2030 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false, 2031 unsigned Depth = 0) const; 2032 2033 /// Return true if this function can prove that \p Op is never poison 2034 /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts 2035 /// argument limits the check to the requested vector elements. 2036 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts, 2037 bool PoisonOnly = false, 2038 unsigned Depth = 0) const; 2039 2040 /// Return true if this function can prove that \p Op is never poison. 2041 bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const { 2042 return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth); 2043 } 2044 2045 /// Return true if this function can prove that \p Op is never poison. The 2046 /// DemandedElts argument limits the check to the requested vector elements. 2047 bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts, 2048 unsigned Depth = 0) const { 2049 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, 2050 /*PoisonOnly*/ true, Depth); 2051 } 2052 2053 /// Return true if Op can create undef or poison from non-undef & non-poison 2054 /// operands. The DemandedElts argument limits the check to the requested 2055 /// vector elements. 2056 /// 2057 /// \p ConsiderFlags controls whether poison producing flags on the 2058 /// instruction are considered. This can be used to see if the instruction 2059 /// could still introduce undef or poison even without poison generating flags 2060 /// which might be on the instruction. (i.e. could the result of 2061 /// Op->dropPoisonGeneratingFlags() still create poison or undef) 2062 bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, 2063 bool PoisonOnly = false, 2064 bool ConsiderFlags = true, 2065 unsigned Depth = 0) const; 2066 2067 /// Return true if Op can create undef or poison from non-undef & non-poison 2068 /// operands. 2069 /// 2070 /// \p ConsiderFlags controls whether poison producing flags on the 2071 /// instruction are considered. This can be used to see if the instruction 2072 /// could still introduce undef or poison even without poison generating flags 2073 /// which might be on the instruction. (i.e. could the result of 2074 /// Op->dropPoisonGeneratingFlags() still create poison or undef) 2075 bool canCreateUndefOrPoison(SDValue Op, bool PoisonOnly = false, 2076 bool ConsiderFlags = true, 2077 unsigned Depth = 0) const; 2078 2079 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode 2080 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that 2081 /// is guaranteed to have the same semantics as an ADD. This handles the 2082 /// equivalence: 2083 /// X|Cst == X+Cst iff X&Cst = 0. 2084 bool isBaseWithConstantOffset(SDValue Op) const; 2085 2086 /// Test whether the given SDValue (or all elements of it, if it is a 2087 /// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is 2088 /// known to never be a signaling NaN (it may still be a qNaN). 2089 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; 2090 2091 /// \returns true if \p Op is known to never be a signaling NaN. 2092 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { 2093 return isKnownNeverNaN(Op, true, Depth); 2094 } 2095 2096 /// Test whether the given floating point SDValue is known to never be 2097 /// positive or negative zero. 2098 bool isKnownNeverZeroFloat(SDValue Op) const; 2099 2100 /// Test whether the given SDValue is known to contain non-zero value(s). 2101 bool isKnownNeverZero(SDValue Op) const; 2102 2103 /// Test whether two SDValues are known to compare equal. This 2104 /// is true if they are the same value, or if one is negative zero and the 2105 /// other positive zero. 2106 bool isEqualTo(SDValue A, SDValue B) const; 2107 2108 /// Return true if A and B have no common bits set. As an example, this can 2109 /// allow an 'add' to be transformed into an 'or'. 2110 bool haveNoCommonBitsSet(SDValue A, SDValue B) const; 2111 2112 /// Test whether \p V has a splatted value for all the demanded elements. 2113 /// 2114 /// On success \p UndefElts will indicate the elements that have UNDEF 2115 /// values instead of the splat value, this is only guaranteed to be correct 2116 /// for \p DemandedElts. 2117 /// 2118 /// NOTE: The function will return true for a demanded splat of UNDEF values. 2119 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, 2120 unsigned Depth = 0) const; 2121 2122 /// Test whether \p V has a splatted value. 2123 bool isSplatValue(SDValue V, bool AllowUndefs = false) const; 2124 2125 /// If V is a splatted value, return the source vector and its splat index. 2126 SDValue getSplatSourceVector(SDValue V, int &SplatIndex); 2127 2128 /// If V is a splat vector, return its scalar source operand by extracting 2129 /// that element from the source vector. If LegalTypes is true, this method 2130 /// may only return a legally-typed splat value. If it cannot legalize the 2131 /// splatted value it will return SDValue(). 2132 SDValue getSplatValue(SDValue V, bool LegalTypes = false); 2133 2134 /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount 2135 /// that is less than the element bit-width of the shift node, return it. 2136 const APInt *getValidShiftAmountConstant(SDValue V, 2137 const APInt &DemandedElts) const; 2138 2139 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 2140 /// than the element bit-width of the shift node, return the minimum value. 2141 const APInt * 2142 getValidMinimumShiftAmountConstant(SDValue V, 2143 const APInt &DemandedElts) const; 2144 2145 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 2146 /// than the element bit-width of the shift node, return the maximum value. 2147 const APInt * 2148 getValidMaximumShiftAmountConstant(SDValue V, 2149 const APInt &DemandedElts) const; 2150 2151 /// Match a binop + shuffle pyramid that represents a horizontal reduction 2152 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p 2153 /// Extract. The reduction must use one of the opcodes listed in /p 2154 /// CandidateBinOps and on success /p BinOp will contain the matching opcode. 2155 /// Returns the vector that is being reduced on, or SDValue() if a reduction 2156 /// was not matched. If \p AllowPartials is set then in the case of a 2157 /// reduction pattern that only matches the first few stages, the extracted 2158 /// subvector of the start of the reduction is returned. 2159 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, 2160 ArrayRef<ISD::NodeType> CandidateBinOps, 2161 bool AllowPartials = false); 2162 2163 /// Utility function used by legalize and lowering to 2164 /// "unroll" a vector operation by splitting out the scalars and operating 2165 /// on each element individually. If the ResNE is 0, fully unroll the vector 2166 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. 2167 /// If the ResNE is greater than the width of the vector op, unroll the 2168 /// vector op and fill the end of the resulting vector with UNDEFS. 2169 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); 2170 2171 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes. 2172 /// This is a separate function because those opcodes have two results. 2173 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N, 2174 unsigned ResNE = 0); 2175 2176 /// Return true if loads are next to each other and can be 2177 /// merged. Check that both are nonvolatile and if LD is loading 2178 /// 'Bytes' bytes from a location that is 'Dist' units away from the 2179 /// location that the 'Base' load is loading from. 2180 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, 2181 unsigned Bytes, int Dist) const; 2182 2183 /// Infer alignment of a load / store address. Return std::nullopt if it 2184 /// cannot be inferred. 2185 MaybeAlign InferPtrAlign(SDValue Ptr) const; 2186 2187 /// Compute the VTs needed for the low/hi parts of a type 2188 /// which is split (or expanded) into two not necessarily identical pieces. 2189 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const; 2190 2191 /// Compute the VTs needed for the low/hi parts of a type, dependent on an 2192 /// enveloping VT that has been split into two identical pieces. Sets the 2193 /// HisIsEmpty flag when hi type has zero storage size. 2194 std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, 2195 bool *HiIsEmpty) const; 2196 2197 /// Split the vector with EXTRACT_SUBVECTOR using the provides 2198 /// VTs and return the low/high part. 2199 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL, 2200 const EVT &LoVT, const EVT &HiVT); 2201 2202 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part. 2203 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { 2204 EVT LoVT, HiVT; 2205 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); 2206 return SplitVector(N, DL, LoVT, HiVT); 2207 } 2208 2209 /// Split the explicit vector length parameter of a VP operation. 2210 std::pair<SDValue, SDValue> SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL); 2211 2212 /// Split the node's operand with EXTRACT_SUBVECTOR and 2213 /// return the low/high part. 2214 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) 2215 { 2216 return SplitVector(N->getOperand(OpNo), SDLoc(N)); 2217 } 2218 2219 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 2220 SDValue WidenVector(const SDValue &N, const SDLoc &DL); 2221 2222 /// Append the extracted elements from Start to Count out of the vector Op in 2223 /// Args. If Count is 0, all of the elements will be extracted. The extracted 2224 /// elements will have type EVT if it is provided, and otherwise their type 2225 /// will be Op's element type. 2226 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, 2227 unsigned Start = 0, unsigned Count = 0, 2228 EVT EltVT = EVT()); 2229 2230 /// Compute the default alignment value for the given type. 2231 Align getEVTAlign(EVT MemoryVT) const; 2232 2233 /// Test whether the given value is a constant int or similar node. 2234 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const; 2235 2236 /// Test whether the given value is a constant FP or similar node. 2237 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ; 2238 2239 /// \returns true if \p N is any kind of constant or build_vector of 2240 /// constants, int or float. If a vector, it may not necessarily be a splat. 2241 inline bool isConstantValueOfAnyType(SDValue N) const { 2242 return isConstantIntBuildVectorOrConstantInt(N) || 2243 isConstantFPBuildVectorOrConstantFP(N); 2244 } 2245 2246 /// Set CallSiteInfo to be associated with Node. 2247 void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo) { 2248 SDEI[Node].CSInfo = std::move(CallInfo); 2249 } 2250 /// Return CallSiteInfo associated with Node, or a default if none exists. 2251 CallSiteInfo getCallSiteInfo(const SDNode *Node) { 2252 auto I = SDEI.find(Node); 2253 return I != SDEI.end() ? std::move(I->second).CSInfo : CallSiteInfo(); 2254 } 2255 /// Set HeapAllocSite to be associated with Node. 2256 void addHeapAllocSite(const SDNode *Node, MDNode *MD) { 2257 SDEI[Node].HeapAllocSite = MD; 2258 } 2259 /// Return HeapAllocSite associated with Node, or nullptr if none exists. 2260 MDNode *getHeapAllocSite(const SDNode *Node) const { 2261 auto I = SDEI.find(Node); 2262 return I != SDEI.end() ? I->second.HeapAllocSite : nullptr; 2263 } 2264 /// Set PCSections to be associated with Node. 2265 void addPCSections(const SDNode *Node, MDNode *MD) { 2266 SDEI[Node].PCSections = MD; 2267 } 2268 /// Return PCSections associated with Node, or nullptr if none exists. 2269 MDNode *getPCSections(const SDNode *Node) const { 2270 auto It = SDEI.find(Node); 2271 return It != SDEI.end() ? It->second.PCSections : nullptr; 2272 } 2273 /// Set NoMergeSiteInfo to be associated with Node if NoMerge is true. 2274 void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { 2275 if (NoMerge) 2276 SDEI[Node].NoMerge = NoMerge; 2277 } 2278 /// Return NoMerge info associated with Node. 2279 bool getNoMergeSiteInfo(const SDNode *Node) const { 2280 auto I = SDEI.find(Node); 2281 return I != SDEI.end() ? I->second.NoMerge : false; 2282 } 2283 2284 /// Copy extra info associated with one node to another. 2285 void copyExtraInfo(SDNode *From, SDNode *To); 2286 2287 /// Return the current function's default denormal handling kind for the given 2288 /// floating point type. 2289 DenormalMode getDenormalMode(EVT VT) const { 2290 return MF->getDenormalMode(EVTToAPFloatSemantics(VT)); 2291 } 2292 2293 bool shouldOptForSize() const; 2294 2295 /// Get the (commutative) neutral element for the given opcode, if it exists. 2296 SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, 2297 SDNodeFlags Flags); 2298 2299 /// Some opcodes may create immediate undefined behavior when used with some 2300 /// values (integer division-by-zero for example). Therefore, these operations 2301 /// are not generally safe to move around or change. 2302 bool isSafeToSpeculativelyExecute(unsigned Opcode) const { 2303 switch (Opcode) { 2304 case ISD::SDIV: 2305 case ISD::SREM: 2306 case ISD::SDIVREM: 2307 case ISD::UDIV: 2308 case ISD::UREM: 2309 case ISD::UDIVREM: 2310 return false; 2311 default: 2312 return true; 2313 } 2314 } 2315 2316 private: 2317 void InsertNode(SDNode *N); 2318 bool RemoveNodeFromCSEMaps(SDNode *N); 2319 void AddModifiedNodeToCSEMaps(SDNode *N); 2320 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos); 2321 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2, 2322 void *&InsertPos); 2323 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 2324 void *&InsertPos); 2325 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc); 2326 2327 void DeleteNodeNotInCSEMaps(SDNode *N); 2328 void DeallocateNode(SDNode *N); 2329 2330 void allnodes_clear(); 2331 2332 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2333 /// not, return the insertion token that will make insertion faster. This 2334 /// overload is for nodes other than Constant or ConstantFP, use the other one 2335 /// for those. 2336 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos); 2337 2338 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2339 /// not, return the insertion token that will make insertion faster. Performs 2340 /// additional processing for constant nodes. 2341 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL, 2342 void *&InsertPos); 2343 2344 /// Maps to auto-CSE operations. 2345 std::vector<CondCodeSDNode*> CondCodeNodes; 2346 2347 std::vector<SDNode*> ValueTypeNodes; 2348 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes; 2349 StringMap<SDNode*> ExternalSymbols; 2350 2351 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols; 2352 DenseMap<MCSymbol *, SDNode *> MCSymbols; 2353 2354 FlagInserter *Inserter = nullptr; 2355 }; 2356 2357 template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> { 2358 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>; 2359 2360 static nodes_iterator nodes_begin(SelectionDAG *G) { 2361 return nodes_iterator(G->allnodes_begin()); 2362 } 2363 2364 static nodes_iterator nodes_end(SelectionDAG *G) { 2365 return nodes_iterator(G->allnodes_end()); 2366 } 2367 }; 2368 2369 } // end namespace llvm 2370 2371 #endif // LLVM_CODEGEN_SELECTIONDAG_H 2372