1 //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file declares the SelectionDAG class, and transitively defines the 10 // SDNode class and subclasses. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_SELECTIONDAG_H 15 #define LLVM_CODEGEN_SELECTIONDAG_H 16 17 #include "llvm/ADT/APFloat.h" 18 #include "llvm/ADT/APInt.h" 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/DenseSet.h" 22 #include "llvm/ADT/FoldingSet.h" 23 #include "llvm/ADT/SetVector.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/StringMap.h" 26 #include "llvm/ADT/ilist.h" 27 #include "llvm/ADT/iterator.h" 28 #include "llvm/ADT/iterator_range.h" 29 #include "llvm/CodeGen/DAGCombine.h" 30 #include "llvm/CodeGen/FunctionLoweringInfo.h" 31 #include "llvm/CodeGen/ISDOpcodes.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineMemOperand.h" 34 #include "llvm/CodeGen/SelectionDAGNodes.h" 35 #include "llvm/CodeGen/ValueTypes.h" 36 #include "llvm/IR/DebugLoc.h" 37 #include "llvm/IR/Instructions.h" 38 #include "llvm/IR/Metadata.h" 39 #include "llvm/Support/Allocator.h" 40 #include "llvm/Support/ArrayRecycler.h" 41 #include "llvm/Support/AtomicOrdering.h" 42 #include "llvm/Support/Casting.h" 43 #include "llvm/Support/CheriSetBounds.h" 44 #include "llvm/Support/CodeGen.h" 45 #include "llvm/Support/ErrorHandling.h" 46 #include "llvm/Support/MachineValueType.h" 47 #include "llvm/Support/RecyclingAllocator.h" 48 #include <algorithm> 49 #include <cassert> 50 #include <cstdint> 51 #include <functional> 52 #include <map> 53 #include <string> 54 #include <tuple> 55 #include <utility> 56 #include <vector> 57 58 namespace llvm { 59 60 class AAResults; 61 class BlockAddress; 62 class BlockFrequencyInfo; 63 class Constant; 64 class ConstantFP; 65 class ConstantInt; 66 class DataLayout; 67 struct fltSemantics; 68 class GlobalValue; 69 struct KnownBits; 70 class LegacyDivergenceAnalysis; 71 class LLVMContext; 72 class MachineBasicBlock; 73 class MachineConstantPoolValue; 74 class MCSymbol; 75 class OptimizationRemarkEmitter; 76 class ProfileSummaryInfo; 77 class SDDbgValue; 78 class SDDbgLabel; 79 class SelectionDAG; 80 class SelectionDAGTargetInfo; 81 class TargetLibraryInfo; 82 class TargetLowering; 83 class TargetMachine; 84 class TargetSubtargetInfo; 85 class Value; 86 87 class SDVTListNode : public FoldingSetNode { 88 friend struct FoldingSetTrait<SDVTListNode>; 89 90 /// A reference to an Interned FoldingSetNodeID for this node. 91 /// The Allocator in SelectionDAG holds the data. 92 /// SDVTList contains all types which are frequently accessed in SelectionDAG. 93 /// The size of this list is not expected to be big so it won't introduce 94 /// a memory penalty. 95 FoldingSetNodeIDRef FastID; 96 const EVT *VTs; 97 unsigned int NumVTs; 98 /// The hash value for SDVTList is fixed, so cache it to avoid 99 /// hash calculation. 100 unsigned HashValue; 101 102 public: 103 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : 104 FastID(ID), VTs(VT), NumVTs(Num) { 105 HashValue = ID.ComputeHash(); 106 } 107 108 SDVTList getSDVTList() { 109 SDVTList result = {VTs, NumVTs}; 110 return result; 111 } 112 }; 113 114 /// Specialize FoldingSetTrait for SDVTListNode 115 /// to avoid computing temp FoldingSetNodeID and hash value. 116 template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> { 117 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { 118 ID = X.FastID; 119 } 120 121 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, 122 unsigned IDHash, FoldingSetNodeID &TempID) { 123 if (X.HashValue != IDHash) 124 return false; 125 return ID == X.FastID; 126 } 127 128 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) { 129 return X.HashValue; 130 } 131 }; 132 133 template <> struct ilist_alloc_traits<SDNode> { 134 static void deleteNode(SDNode *) { 135 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!"); 136 } 137 }; 138 139 /// Keeps track of dbg_value information through SDISel. We do 140 /// not build SDNodes for these so as not to perturb the generated code; 141 /// instead the info is kept off to the side in this structure. Each SDNode may 142 /// have one or more associated dbg_value entries. This information is kept in 143 /// DbgValMap. 144 /// Byval parameters are handled separately because they don't use alloca's, 145 /// which busts the normal mechanism. There is good reason for handling all 146 /// parameters separately: they may not have code generated for them, they 147 /// should always go at the beginning of the function regardless of other code 148 /// motion, and debug info for them is potentially useful even if the parameter 149 /// is unused. Right now only byval parameters are handled separately. 150 class SDDbgInfo { 151 BumpPtrAllocator Alloc; 152 SmallVector<SDDbgValue*, 32> DbgValues; 153 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; 154 SmallVector<SDDbgLabel*, 4> DbgLabels; 155 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>; 156 DbgValMapType DbgValMap; 157 158 public: 159 SDDbgInfo() = default; 160 SDDbgInfo(const SDDbgInfo &) = delete; 161 SDDbgInfo &operator=(const SDDbgInfo &) = delete; 162 163 void add(SDDbgValue *V, const SDNode *Node, bool isParameter) { 164 if (isParameter) { 165 ByvalParmDbgValues.push_back(V); 166 } else DbgValues.push_back(V); 167 if (Node) 168 DbgValMap[Node].push_back(V); 169 } 170 171 void add(SDDbgLabel *L) { 172 DbgLabels.push_back(L); 173 } 174 175 /// Invalidate all DbgValues attached to the node and remove 176 /// it from the Node-to-DbgValues map. 177 void erase(const SDNode *Node); 178 179 void clear() { 180 DbgValMap.clear(); 181 DbgValues.clear(); 182 ByvalParmDbgValues.clear(); 183 DbgLabels.clear(); 184 Alloc.Reset(); 185 } 186 187 BumpPtrAllocator &getAlloc() { return Alloc; } 188 189 bool empty() const { 190 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); 191 } 192 193 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const { 194 auto I = DbgValMap.find(Node); 195 if (I != DbgValMap.end()) 196 return I->second; 197 return ArrayRef<SDDbgValue*>(); 198 } 199 200 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator; 201 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator; 202 203 DbgIterator DbgBegin() { return DbgValues.begin(); } 204 DbgIterator DbgEnd() { return DbgValues.end(); } 205 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); } 206 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); } 207 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); } 208 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); } 209 }; 210 211 void checkForCycles(const SelectionDAG *DAG, bool force = false); 212 213 /// This is used to represent a portion of an LLVM function in a low-level 214 /// Data Dependence DAG representation suitable for instruction selection. 215 /// This DAG is constructed as the first step of instruction selection in order 216 /// to allow implementation of machine specific optimizations 217 /// and code simplifications. 218 /// 219 /// The representation used by the SelectionDAG is a target-independent 220 /// representation, which has some similarities to the GCC RTL representation, 221 /// but is significantly more simple, powerful, and is a graph form instead of a 222 /// linear form. 223 /// 224 class SelectionDAG { 225 const TargetMachine &TM; 226 const SelectionDAGTargetInfo *TSI = nullptr; 227 const TargetLowering *TLI = nullptr; 228 const TargetLibraryInfo *LibInfo = nullptr; 229 MachineFunction *MF; 230 Pass *SDAGISelPass = nullptr; 231 LLVMContext *Context; 232 CodeGenOpt::Level OptLevel; 233 234 LegacyDivergenceAnalysis * DA = nullptr; 235 FunctionLoweringInfo * FLI = nullptr; 236 237 /// The function-level optimization remark emitter. Used to emit remarks 238 /// whenever manipulating the DAG. 239 OptimizationRemarkEmitter *ORE; 240 241 ProfileSummaryInfo *PSI = nullptr; 242 BlockFrequencyInfo *BFI = nullptr; 243 244 /// The starting token. 245 SDNode EntryNode; 246 247 /// The root of the entire DAG. 248 SDValue Root; 249 250 /// A linked list of nodes in the current DAG. 251 ilist<SDNode> AllNodes; 252 253 /// The AllocatorType for allocating SDNodes. We use 254 /// pool allocation with recycling. 255 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode, 256 sizeof(LargestSDNode), 257 alignof(MostAlignedSDNode)>; 258 259 /// Pool allocation for nodes. 260 NodeAllocatorType NodeAllocator; 261 262 /// This structure is used to memoize nodes, automatically performing 263 /// CSE with existing nodes when a duplicate is requested. 264 FoldingSet<SDNode> CSEMap; 265 266 /// Pool allocation for machine-opcode SDNode operands. 267 BumpPtrAllocator OperandAllocator; 268 ArrayRecycler<SDUse> OperandRecycler; 269 270 /// Pool allocation for misc. objects that are created once per SelectionDAG. 271 BumpPtrAllocator Allocator; 272 273 /// Tracks dbg_value and dbg_label information through SDISel. 274 SDDbgInfo *DbgInfo; 275 276 using CallSiteInfo = MachineFunction::CallSiteInfo; 277 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; 278 279 struct CallSiteDbgInfo { 280 CallSiteInfo CSInfo; 281 MDNode *HeapAllocSite = nullptr; 282 bool NoMerge = false; 283 }; 284 285 DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo; 286 287 uint16_t NextPersistentId = 0; 288 289 public: 290 /// Clients of various APIs that cause global effects on 291 /// the DAG can optionally implement this interface. This allows the clients 292 /// to handle the various sorts of updates that happen. 293 /// 294 /// A DAGUpdateListener automatically registers itself with DAG when it is 295 /// constructed, and removes itself when destroyed in RAII fashion. 296 struct DAGUpdateListener { 297 DAGUpdateListener *const Next; 298 SelectionDAG &DAG; 299 300 explicit DAGUpdateListener(SelectionDAG &D) 301 : Next(D.UpdateListeners), DAG(D) { 302 DAG.UpdateListeners = this; 303 } 304 305 virtual ~DAGUpdateListener() { 306 assert(DAG.UpdateListeners == this && 307 "DAGUpdateListeners must be destroyed in LIFO order"); 308 DAG.UpdateListeners = Next; 309 } 310 311 /// The node N that was deleted and, if E is not null, an 312 /// equivalent node E that replaced it. 313 virtual void NodeDeleted(SDNode *N, SDNode *E); 314 315 /// The node N that was updated. 316 virtual void NodeUpdated(SDNode *N); 317 318 /// The node N that was inserted. 319 virtual void NodeInserted(SDNode *N); 320 }; 321 322 struct DAGNodeDeletedListener : public DAGUpdateListener { 323 std::function<void(SDNode *, SDNode *)> Callback; 324 325 DAGNodeDeletedListener(SelectionDAG &DAG, 326 std::function<void(SDNode *, SDNode *)> Callback) 327 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} 328 329 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } 330 331 private: 332 virtual void anchor(); 333 }; 334 335 /// When true, additional steps are taken to 336 /// ensure that getConstant() and similar functions return DAG nodes that 337 /// have legal types. This is important after type legalization since 338 /// any illegally typed nodes generated after this point will not experience 339 /// type legalization. 340 bool NewNodesMustHaveLegalTypes = false; 341 342 private: 343 /// DAGUpdateListener is a friend so it can manipulate the listener stack. 344 friend struct DAGUpdateListener; 345 346 /// Linked list of registered DAGUpdateListener instances. 347 /// This stack is maintained by DAGUpdateListener RAII. 348 DAGUpdateListener *UpdateListeners = nullptr; 349 350 /// Implementation of setSubgraphColor. 351 /// Return whether we had to truncate the search. 352 bool setSubgraphColorHelper(SDNode *N, const char *Color, 353 DenseSet<SDNode *> &visited, 354 int level, bool &printed); 355 356 template <typename SDNodeT, typename... ArgTypes> 357 SDNodeT *newSDNode(ArgTypes &&... Args) { 358 return new (NodeAllocator.template Allocate<SDNodeT>()) 359 SDNodeT(std::forward<ArgTypes>(Args)...); 360 } 361 362 /// Build a synthetic SDNodeT with the given args and extract its subclass 363 /// data as an integer (e.g. for use in a folding set). 364 /// 365 /// The args to this function are the same as the args to SDNodeT's 366 /// constructor, except the second arg (assumed to be a const DebugLoc&) is 367 /// omitted. 368 template <typename SDNodeT, typename... ArgTypes> 369 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder, 370 ArgTypes &&... Args) { 371 // The compiler can reduce this expression to a constant iff we pass an 372 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing 373 // on the subclass data. 374 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...) 375 .getRawSubclassData(); 376 } 377 378 template <typename SDNodeTy> 379 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order, 380 SDVTList VTs, EVT MemoryVT, 381 MachineMemOperand *MMO) { 382 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO) 383 .getRawSubclassData(); 384 } 385 386 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals); 387 388 void removeOperands(SDNode *Node) { 389 if (!Node->OperandList) 390 return; 391 OperandRecycler.deallocate( 392 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands), 393 Node->OperandList); 394 Node->NumOperands = 0; 395 Node->OperandList = nullptr; 396 } 397 void CreateTopologicalOrder(std::vector<SDNode*>& Order); 398 399 public: 400 // Maximum depth for recursive analysis such as computeKnownBits, etc. 401 static constexpr unsigned MaxRecursionDepth = 6; 402 403 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); 404 SelectionDAG(const SelectionDAG &) = delete; 405 SelectionDAG &operator=(const SelectionDAG &) = delete; 406 ~SelectionDAG(); 407 408 /// Prepare this SelectionDAG to process code in the given MachineFunction. 409 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, 410 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 411 LegacyDivergenceAnalysis * Divergence, 412 ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin); 413 414 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { 415 FLI = FuncInfo; 416 } 417 418 /// Clear state and free memory necessary to make this 419 /// SelectionDAG ready to process a new block. 420 void clear(); 421 422 MachineFunction &getMachineFunction() const { return *MF; } 423 const Pass *getPass() const { return SDAGISelPass; } 424 425 const DataLayout &getDataLayout() const { return MF->getDataLayout(); } 426 const TargetMachine &getTarget() const { return TM; } 427 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } 428 const TargetLowering &getTargetLoweringInfo() const { return *TLI; } 429 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } 430 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } 431 const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } 432 LLVMContext *getContext() const { return Context; } 433 OptimizationRemarkEmitter &getORE() const { return *ORE; } 434 ProfileSummaryInfo *getPSI() const { return PSI; } 435 BlockFrequencyInfo *getBFI() const { return BFI; } 436 437 /// Just dump dot graph to a user-provided path and title. 438 /// This doesn't open the dot viewer program and 439 /// helps visualization when outside debugging session. 440 /// FileName expects absolute path. If provided 441 /// without any path separators then the file 442 /// will be created in the current directory. 443 /// Error will be emitted if the path is insane. 444 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 445 LLVM_DUMP_METHOD void dumpDotGraph(const Twine &FileName, const Twine &Title); 446 #endif 447 448 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. 449 void viewGraph(const std::string &Title); 450 void viewGraph(); 451 452 #ifndef NDEBUG 453 std::map<const SDNode *, std::string> NodeGraphAttrs; 454 #endif 455 456 /// Clear all previously defined node graph attributes. 457 /// Intended to be used from a debugging tool (eg. gdb). 458 void clearGraphAttrs(); 459 460 /// Set graph attributes for a node. (eg. "color=red".) 461 void setGraphAttrs(const SDNode *N, const char *Attrs); 462 463 /// Get graph attributes for a node. (eg. "color=red".) 464 /// Used from getNodeAttributes. 465 const std::string getGraphAttrs(const SDNode *N) const; 466 467 /// Convenience for setting node color attribute. 468 void setGraphColor(const SDNode *N, const char *Color); 469 470 /// Convenience for setting subgraph color attribute. 471 void setSubgraphColor(SDNode *N, const char *Color); 472 473 using allnodes_const_iterator = ilist<SDNode>::const_iterator; 474 475 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); } 476 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); } 477 478 using allnodes_iterator = ilist<SDNode>::iterator; 479 480 allnodes_iterator allnodes_begin() { return AllNodes.begin(); } 481 allnodes_iterator allnodes_end() { return AllNodes.end(); } 482 483 ilist<SDNode>::size_type allnodes_size() const { 484 return AllNodes.size(); 485 } 486 487 iterator_range<allnodes_iterator> allnodes() { 488 return make_range(allnodes_begin(), allnodes_end()); 489 } 490 iterator_range<allnodes_const_iterator> allnodes() const { 491 return make_range(allnodes_begin(), allnodes_end()); 492 } 493 494 /// Return the root tag of the SelectionDAG. 495 const SDValue &getRoot() const { return Root; } 496 497 /// Return the token chain corresponding to the entry of the function. 498 SDValue getEntryNode() const { 499 return SDValue(const_cast<SDNode *>(&EntryNode), 0); 500 } 501 502 /// Set the current root tag of the SelectionDAG. 503 /// 504 const SDValue &setRoot(SDValue N) { 505 assert((!N.getNode() || N.getValueType() == MVT::Other) && 506 "DAG root value is not a chain!"); 507 if (N.getNode()) 508 checkForCycles(N.getNode(), this); 509 Root = N; 510 if (N.getNode()) 511 checkForCycles(this); 512 return Root; 513 } 514 515 #ifndef NDEBUG 516 void VerifyDAGDiverence(); 517 #endif 518 519 /// This iterates over the nodes in the SelectionDAG, folding 520 /// certain types of nodes together, or eliminating superfluous nodes. The 521 /// Level argument controls whether Combine is allowed to produce nodes and 522 /// types that are illegal on the target. 523 void Combine(CombineLevel Level, AAResults *AA, 524 CodeGenOpt::Level OptLevel); 525 526 /// This transforms the SelectionDAG into a SelectionDAG that 527 /// only uses types natively supported by the target. 528 /// Returns "true" if it made any changes. 529 /// 530 /// Note that this is an involved process that may invalidate pointers into 531 /// the graph. 532 bool LegalizeTypes(); 533 534 /// This transforms the SelectionDAG into a SelectionDAG that is 535 /// compatible with the target instruction selector, as indicated by the 536 /// TargetLowering object. 537 /// 538 /// Note that this is an involved process that may invalidate pointers into 539 /// the graph. 540 void Legalize(); 541 542 /// Transforms a SelectionDAG node and any operands to it into a node 543 /// that is compatible with the target instruction selector, as indicated by 544 /// the TargetLowering object. 545 /// 546 /// \returns true if \c N is a valid, legal node after calling this. 547 /// 548 /// This essentially runs a single recursive walk of the \c Legalize process 549 /// over the given node (and its operands). This can be used to incrementally 550 /// legalize the DAG. All of the nodes which are directly replaced, 551 /// potentially including N, are added to the output parameter \c 552 /// UpdatedNodes so that the delta to the DAG can be understood by the 553 /// caller. 554 /// 555 /// When this returns false, N has been legalized in a way that make the 556 /// pointer passed in no longer valid. It may have even been deleted from the 557 /// DAG, and so it shouldn't be used further. When this returns true, the 558 /// N passed in is a legal node, and can be immediately processed as such. 559 /// This may still have done some work on the DAG, and will still populate 560 /// UpdatedNodes with any new nodes replacing those originally in the DAG. 561 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes); 562 563 /// This transforms the SelectionDAG into a SelectionDAG 564 /// that only uses vector math operations supported by the target. This is 565 /// necessary as a separate step from Legalize because unrolling a vector 566 /// operation can introduce illegal types, which requires running 567 /// LegalizeTypes again. 568 /// 569 /// This returns true if it made any changes; in that case, LegalizeTypes 570 /// is called again before Legalize. 571 /// 572 /// Note that this is an involved process that may invalidate pointers into 573 /// the graph. 574 bool LegalizeVectors(); 575 576 /// This method deletes all unreachable nodes in the SelectionDAG. 577 void RemoveDeadNodes(); 578 579 /// Remove the specified node from the system. This node must 580 /// have no referrers. 581 void DeleteNode(SDNode *N); 582 583 /// Return an SDVTList that represents the list of values specified. 584 SDVTList getVTList(EVT VT); 585 SDVTList getVTList(EVT VT1, EVT VT2); 586 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3); 587 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4); 588 SDVTList getVTList(ArrayRef<EVT> VTs); 589 590 //===--------------------------------------------------------------------===// 591 // Node creation methods. 592 593 /// Create a ConstantSDNode wrapping a constant value. 594 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 595 /// 596 /// If only legal types can be produced, this does the necessary 597 /// transformations (e.g., if the vector element type is illegal). 598 /// @{ 599 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 600 bool isTarget = false, bool isOpaque = false); 601 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 602 bool isTarget = false, bool isOpaque = false); 603 604 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false, 605 bool IsOpaque = false) { 606 return getConstant(APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL, 607 VT, IsTarget, IsOpaque); 608 } 609 610 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 611 bool isTarget = false, bool isOpaque = false); 612 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, 613 bool isTarget = false); 614 SDValue getNullCapability(const SDLoc &DL); 615 616 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, 617 bool LegalTypes = true); 618 SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, 619 bool isTarget = false); 620 621 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, 622 bool isOpaque = false) { 623 return getConstant(Val, DL, VT, true, isOpaque); 624 } 625 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT, 626 bool isOpaque = false) { 627 return getConstant(Val, DL, VT, true, isOpaque); 628 } 629 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 630 bool isOpaque = false) { 631 return getConstant(Val, DL, VT, true, isOpaque); 632 } 633 634 /// Create a true or false constant of type \p VT using the target's 635 /// BooleanContent for type \p OpVT. 636 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT); 637 /// @} 638 639 /// Create a ConstantFPSDNode wrapping a constant value. 640 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 641 /// 642 /// If only legal types can be produced, this does the necessary 643 /// transformations (e.g., if the vector element type is illegal). 644 /// The forms that take a double should only be used for simple constants 645 /// that can be exactly represented in VT. No checks are made. 646 /// @{ 647 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, 648 bool isTarget = false); 649 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT, 650 bool isTarget = false); 651 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT, 652 bool isTarget = false); 653 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) { 654 return getConstantFP(Val, DL, VT, true); 655 } 656 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) { 657 return getConstantFP(Val, DL, VT, true); 658 } 659 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) { 660 return getConstantFP(Val, DL, VT, true); 661 } 662 /// @} 663 664 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 665 int64_t offset = 0, bool isTargetGA = false, 666 unsigned TargetFlags = 0); 667 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 668 int64_t offset = 0, unsigned TargetFlags = 0) { 669 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); 670 } 671 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); 672 SDValue getTargetFrameIndex(int FI, EVT VT) { 673 return getFrameIndex(FI, VT, true); 674 } 675 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false, 676 unsigned TargetFlags = 0); 677 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) { 678 return getJumpTable(JTI, VT, true, TargetFlags); 679 } 680 SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align = None, 681 int Offs = 0, bool isT = false, 682 unsigned TargetFlags = 0); 683 SDValue getTargetConstantPool(const Constant *C, EVT VT, 684 MaybeAlign Align = None, int Offset = 0, 685 unsigned TargetFlags = 0) { 686 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 687 } 688 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT, 689 MaybeAlign Align = None, int Offs = 0, 690 bool isT = false, unsigned TargetFlags = 0); 691 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, 692 MaybeAlign Align = None, int Offset = 0, 693 unsigned TargetFlags = 0) { 694 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 695 } 696 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0, 697 unsigned TargetFlags = 0); 698 // When generating a branch to a BB, we don't in general know enough 699 // to provide debug info for the BB at that time, so keep this one around. 700 SDValue getBasicBlock(MachineBasicBlock *MBB); 701 SDValue getBasicBlock(MachineBasicBlock *MBB, SDLoc dl); 702 SDValue getExternalSymbol(const char *Sym, EVT VT); 703 SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT); 704 SDValue getTargetExternalSymbol(const char *Sym, EVT VT, 705 unsigned TargetFlags = 0); 706 SDValue getExternalFunctionSymbol(const char *Sym); 707 SDValue getTargetExternalFunctionSymbol(const char *Sym, 708 unsigned TargetFlags = 0); 709 SDValue getMCSymbol(MCSymbol *Sym, EVT VT); 710 711 SDValue getValueType(EVT); 712 SDValue getRegister(unsigned Reg, EVT VT); 713 SDValue getRegisterMask(const uint32_t *RegMask); 714 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); 715 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, 716 MCSymbol *Label); 717 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, 718 bool isTarget = false, unsigned TargetFlags = 0); 719 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, 720 int64_t Offset = 0, unsigned TargetFlags = 0) { 721 return getBlockAddress(BA, VT, Offset, true, TargetFlags); 722 } 723 724 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, 725 SDValue N) { 726 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain, 727 getRegister(Reg, N.getValueType()), N); 728 } 729 730 // This version of the getCopyToReg method takes an extra operand, which 731 // indicates that there is potentially an incoming glue value (if Glue is not 732 // null) and that there should be a glue result. 733 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N, 734 SDValue Glue) { 735 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 736 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue }; 737 return getNode(ISD::CopyToReg, dl, VTs, 738 makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); 739 } 740 741 // Similar to last getCopyToReg() except parameter Reg is a SDValue 742 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N, 743 SDValue Glue) { 744 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 745 SDValue Ops[] = { Chain, Reg, N, Glue }; 746 return getNode(ISD::CopyToReg, dl, VTs, 747 makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); 748 } 749 750 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) { 751 SDVTList VTs = getVTList(VT, MVT::Other); 752 SDValue Ops[] = { Chain, getRegister(Reg, VT) }; 753 return getNode(ISD::CopyFromReg, dl, VTs, Ops); 754 } 755 756 // This version of the getCopyFromReg method takes an extra operand, which 757 // indicates that there is potentially an incoming glue value (if Glue is not 758 // null) and that there should be a glue result. 759 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT, 760 SDValue Glue) { 761 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue); 762 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue }; 763 return getNode(ISD::CopyFromReg, dl, VTs, 764 makeArrayRef(Ops, Glue.getNode() ? 3 : 2)); 765 } 766 767 SDValue getCondCode(ISD::CondCode Cond); 768 769 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT, 770 /// which must be a vector type, must match the number of mask elements 771 /// NumElts. An integer mask element equal to -1 is treated as undefined. 772 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, 773 ArrayRef<int> Mask); 774 775 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 776 /// which must be a vector type, must match the number of operands in Ops. 777 /// The operands must have the same type as (or, for integers, a type wider 778 /// than) VT's element type. 779 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) { 780 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 781 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 782 } 783 784 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 785 /// which must be a vector type, must match the number of operands in Ops. 786 /// The operands must have the same type as (or, for integers, a type wider 787 /// than) VT's element type. 788 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) { 789 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 790 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 791 } 792 793 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all 794 /// elements. VT must be a vector type. Op's type must be the same as (or, 795 /// for integers, a type wider than) VT's element type. 796 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) { 797 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 798 if (Op.getOpcode() == ISD::UNDEF) { 799 assert((VT.getVectorElementType() == Op.getValueType() || 800 (VT.isInteger() && 801 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 802 "A splatted value must have a width equal or (for integers) " 803 "greater than the vector element type!"); 804 return getNode(ISD::UNDEF, SDLoc(), VT); 805 } 806 807 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op); 808 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 809 } 810 811 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all 812 // elements. 813 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) { 814 if (Op.getOpcode() == ISD::UNDEF) { 815 assert((VT.getVectorElementType() == Op.getValueType() || 816 (VT.isInteger() && 817 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 818 "A splatted value must have a width equal or (for integers) " 819 "greater than the vector element type!"); 820 return getNode(ISD::UNDEF, SDLoc(), VT); 821 } 822 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op); 823 } 824 825 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to 826 /// the shuffle node in input but with swapped operands. 827 /// 828 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3> 829 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV); 830 831 /// Convert Op, which must be of float type, to the 832 /// float type VT, by either extending or rounding (by truncation). 833 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT); 834 835 /// Convert Op, which must be a STRICT operation of float type, to the 836 /// float type VT, by either extending or rounding (by truncation). 837 std::pair<SDValue, SDValue> 838 getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT); 839 840 /// Convert Op, which must be of integer type, to the 841 /// integer type VT, by either any-extending or truncating it. 842 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 843 844 /// Convert Op, which must be of integer type, to the 845 /// integer type VT, by either sign-extending or truncating it. 846 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 847 848 /// Convert Op, which must be of integer type, to the 849 /// integer type VT, by either zero-extending or truncating it. 850 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 851 852 /// Return the expression required to zero extend the Op 853 /// value assuming it was the smaller SrcTy value. 854 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 855 856 /// Convert Op, which must be of integer type, to the integer type VT, by 857 /// either truncating it or performing either zero or sign extension as 858 /// appropriate extension for the pointer's semantics. 859 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 860 861 /// Return the expression required to extend the Op as a pointer value 862 /// assuming it was the smaller SrcTy value. This may be either a zero extend 863 /// or a sign extend. 864 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 865 866 /// Convert Op, which must be of integer type, to the integer type VT, 867 /// by using an extension appropriate for the target's 868 /// BooleanContent for type OpVT or truncating it. 869 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT); 870 871 /// Create a bitwise NOT operation as (XOR Val, -1). 872 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT); 873 874 /// Create a logical NOT operation as (XOR Val, BooleanOne). 875 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); 876 877 /// Generate a CHERI CSetBounds intrinsic. 878 /// Also create a log record if CSetBounds stats are being gathered 879 SDValue getCSetBounds(SDValue Val, const SDLoc &DL, SDValue Length, 880 Align Alignment, StringRef Pass, 881 cheri::SetBoundsPointerSource Kind, 882 const Twine &Reason = "", std::string SrcLoc = {}); 883 SDValue getCSetBounds(SDValue Val, const SDLoc &DL, uint64_t Length, 884 Align Alignment, StringRef Pass, 885 cheri::SetBoundsPointerSource Kind, 886 const Twine &Reason = "", std::string SrcLoc = {}) { 887 return getCSetBounds(Val, DL, getIntPtrConstant(Length, SDLoc(Val)), 888 Alignment, Pass, Kind, Reason, SrcLoc); 889 } 890 891 // Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default 892 SDValue getPointerAdd(const SDLoc &DL, SDValue Ptr, int64_t Offset, 893 const SDNodeFlags Flags = SDNodeFlags()) { 894 return getMemBasePlusOffset(Ptr, Offset, DL, Flags); 895 } 896 897 // Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default 898 SDValue getPointerAdd(const SDLoc &DL, SDValue Ptr, SDValue Offset, 899 const SDNodeFlags Flags = SDNodeFlags()) { 900 return getMemBasePlusOffset(Ptr, Offset, DL, Flags); 901 } 902 903 /// Returns sum of the base pointer and offset. 904 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. 905 SDValue getMemBasePlusOffset(SDValue Base, int64_t Offset, const SDLoc &DL, 906 const SDNodeFlags Flags = SDNodeFlags()); 907 SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL, 908 const SDNodeFlags Flags = SDNodeFlags()); 909 910 /// Create an add instruction with appropriate flags when used for 911 /// addressing some offset of an object. i.e. if a load is split into multiple 912 /// components, create an add nuw from the base pointer to the offset. 913 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, int64_t Offset) { 914 SDNodeFlags Flags; 915 Flags.setNoUnsignedWrap(true); 916 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 917 } 918 919 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { 920 // The object itself can't wrap around the address space, so it shouldn't be 921 // possible for the adds of the offsets to the split parts to overflow. 922 SDNodeFlags Flags; 923 Flags.setNoUnsignedWrap(true); 924 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 925 } 926 927 /// Return a new CALLSEQ_START node, that starts new call frame, in which 928 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and 929 /// OutSize specifies part of the frame set up prior to the sequence. 930 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, 931 const SDLoc &DL) { 932 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 933 SDValue Ops[] = { Chain, 934 getIntPtrConstant(InSize, DL, true), 935 getIntPtrConstant(OutSize, DL, true) }; 936 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops); 937 } 938 939 /// Return a new CALLSEQ_END node, which always must have a 940 /// glue result (to ensure it's not CSE'd). 941 /// CALLSEQ_END does not have a useful SDLoc. 942 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, 943 SDValue InGlue, const SDLoc &DL) { 944 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue); 945 SmallVector<SDValue, 4> Ops; 946 Ops.push_back(Chain); 947 Ops.push_back(Op1); 948 Ops.push_back(Op2); 949 if (InGlue.getNode()) 950 Ops.push_back(InGlue); 951 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops); 952 } 953 954 /// Return true if the result of this operation is always undefined. 955 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops); 956 957 /// Return an UNDEF node. UNDEF does not have a useful SDLoc. 958 SDValue getUNDEF(EVT VT) { 959 return getNode(ISD::UNDEF, SDLoc(), VT); 960 } 961 962 /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'. 963 SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) { 964 assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() && 965 "Immediate does not fit VT"); 966 return getNode(ISD::VSCALE, DL, VT, 967 getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT)); 968 } 969 970 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc. 971 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) { 972 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); 973 } 974 975 /// Gets or creates the specified node. 976 /// 977 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 978 ArrayRef<SDUse> Ops); 979 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 980 ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags()); 981 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, 982 ArrayRef<SDValue> Ops); 983 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 984 ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags()); 985 986 // Specialize based on number of operands. 987 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT); 988 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, 989 const SDNodeFlags Flags = SDNodeFlags()); 990 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 991 SDValue N2, const SDNodeFlags Flags = SDNodeFlags()); 992 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 993 SDValue N2, SDValue N3, 994 const SDNodeFlags Flags = SDNodeFlags()); 995 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 996 SDValue N2, SDValue N3, SDValue N4); 997 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 998 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 999 1000 // Specialize again based on number of operands for nodes with a VTList 1001 // rather than a single VT. 1002 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList); 1003 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N); 1004 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1005 SDValue N2); 1006 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1007 SDValue N2, SDValue N3); 1008 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1009 SDValue N2, SDValue N3, SDValue N4); 1010 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1011 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 1012 1013 /// Compute a TokenFactor to force all the incoming stack arguments to be 1014 /// loaded from the stack. This is used in tail call lowering to protect 1015 /// stack arguments from being clobbered. 1016 SDValue getStackArgumentTokenFactor(SDValue Chain); 1017 1018 LLVM_ATTRIBUTE_DEPRECATED( 1019 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 1020 SDValue Src, SDValue Size, unsigned Align, bool isVol, 1021 bool AlwaysInline, bool isTailCall, 1022 bool MustPreserveCheriCapabilities, 1023 MachinePointerInfo DstPtrInfo, 1024 MachinePointerInfo SrcPtrInfo, 1025 StringRef CopyType = StringRef()), 1026 "Use the version that takes Align instead") { 1027 return getMemcpy(Chain, dl, Dst, Src, Size, llvm::Align(Align), isVol, 1028 AlwaysInline, isTailCall, MustPreserveCheriCapabilities, 1029 DstPtrInfo, SrcPtrInfo, CopyType); 1030 } 1031 1032 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1033 SDValue Size, Align Alignment, bool isVol, 1034 bool AlwaysInline, bool isTailCall, 1035 bool MustPreserveCheriCapabilities, 1036 MachinePointerInfo DstPtrInfo, 1037 MachinePointerInfo SrcPtrInfo, 1038 StringRef CopyType = StringRef()); 1039 1040 LLVM_ATTRIBUTE_DEPRECATED( 1041 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 1042 SDValue Src, SDValue Size, unsigned Align, bool isVol, 1043 bool isTailCall, bool MustPreserveCheriCapabilities, 1044 MachinePointerInfo DstPtrInfo, 1045 MachinePointerInfo SrcPtrInfo, StringRef MoveType = StringRef()), 1046 "Use the version that takes Align instead") { 1047 return getMemmove(Chain, dl, Dst, Src, Size, llvm::Align(Align), isVol, 1048 isTailCall, MustPreserveCheriCapabilities, DstPtrInfo, 1049 SrcPtrInfo, MoveType); 1050 } 1051 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1052 SDValue Size, Align Alignment, bool isVol, bool isTailCall, 1053 bool MustPreserveCheriCapabilities, 1054 MachinePointerInfo DstPtrInfo, 1055 MachinePointerInfo SrcPtrInfo, 1056 StringRef MoveType = StringRef()); 1057 1058 LLVM_ATTRIBUTE_DEPRECATED(SDValue getMemset(SDValue Chain, const SDLoc &dl, 1059 SDValue Dst, SDValue Src, 1060 SDValue Size, unsigned Align, 1061 bool isVol, bool isTailCall, 1062 MachinePointerInfo DstPtrInfo), 1063 "Use the version that takes Align instead") { 1064 return getMemset(Chain, dl, Dst, Src, Size, llvm::Align(Align), isVol, 1065 isTailCall, DstPtrInfo); 1066 } 1067 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1068 SDValue Size, Align Alignment, bool isVol, bool isTailCall, 1069 MachinePointerInfo DstPtrInfo); 1070 1071 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 1072 unsigned DstAlign, SDValue Src, unsigned SrcAlign, 1073 SDValue Size, Type *SizeTy, unsigned ElemSz, 1074 bool isTailCall, MachinePointerInfo DstPtrInfo, 1075 MachinePointerInfo SrcPtrInfo); 1076 1077 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 1078 unsigned DstAlign, SDValue Src, unsigned SrcAlign, 1079 SDValue Size, Type *SizeTy, unsigned ElemSz, 1080 bool isTailCall, MachinePointerInfo DstPtrInfo, 1081 MachinePointerInfo SrcPtrInfo); 1082 1083 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 1084 unsigned DstAlign, SDValue Value, SDValue Size, 1085 Type *SizeTy, unsigned ElemSz, bool isTailCall, 1086 MachinePointerInfo DstPtrInfo); 1087 1088 /// Helper function to make it easier to build SetCC's if you just have an 1089 /// ISD::CondCode instead of an SDValue. 1090 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, 1091 ISD::CondCode Cond, SDValue Chain = SDValue(), 1092 bool IsSignaling = false) { 1093 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && 1094 "Cannot compare scalars to vectors"); 1095 assert(LHS.getValueType().isVector() == VT.isVector() && 1096 "Cannot compare scalars to vectors"); 1097 assert(Cond != ISD::SETCC_INVALID && 1098 "Cannot create a setCC of an invalid node."); 1099 if (Chain) 1100 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL, 1101 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)}); 1102 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); 1103 } 1104 1105 /// Helper function to make it easier to build Select's if you just have 1106 /// operands and don't want to check for vector. 1107 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, 1108 SDValue RHS) { 1109 assert(LHS.getValueType() == RHS.getValueType() && 1110 "Cannot use select on differing types"); 1111 assert(VT.isVector() == LHS.getValueType().isVector() && 1112 "Cannot mix vectors and scalars"); 1113 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; 1114 return getNode(Opcode, DL, VT, Cond, LHS, RHS); 1115 } 1116 1117 /// Helper function to make it easier to build SelectCC's if you just have an 1118 /// ISD::CondCode instead of an SDValue. 1119 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, 1120 SDValue False, ISD::CondCode Cond) { 1121 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, 1122 False, getCondCode(Cond)); 1123 } 1124 1125 /// Try to simplify a select/vselect into 1 of its operands or a constant. 1126 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); 1127 1128 /// Try to simplify a shift into 1 of its operands or a constant. 1129 SDValue simplifyShift(SDValue X, SDValue Y); 1130 1131 /// Try to simplify a floating-point binary operation into 1 of its operands 1132 /// or a constant. 1133 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, 1134 SDNodeFlags Flags); 1135 1136 /// VAArg produces a result and token chain, and takes a pointer 1137 /// and a source value as input. 1138 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1139 SDValue SV, unsigned Align); 1140 1141 /// Gets a node for an atomic cmpxchg op. There are two 1142 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a 1143 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, 1144 /// a success flag (initially i1), and a chain. 1145 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1146 SDVTList VTs, SDValue Chain, SDValue Ptr, 1147 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); 1148 1149 /// Gets a node for an atomic op, produces result (if relevant) 1150 /// and chain and takes 2 operands. 1151 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, 1152 SDValue Ptr, SDValue Val, MachineMemOperand *MMO); 1153 1154 /// Gets a node for an atomic op, produces result and chain and 1155 /// takes 1 operand. 1156 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, 1157 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); 1158 1159 /// Gets a node for an atomic op, produces result and chain and takes N 1160 /// operands. 1161 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1162 SDVTList VTList, ArrayRef<SDValue> Ops, 1163 MachineMemOperand *MMO); 1164 1165 /// Creates a MemIntrinsicNode that may produce a 1166 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, 1167 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not 1168 /// less than FIRST_TARGET_MEMORY_OPCODE. 1169 SDValue getMemIntrinsicNode( 1170 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1171 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, 1172 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1173 MachineMemOperand::MOStore, 1174 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()); 1175 1176 inline SDValue getMemIntrinsicNode( 1177 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1178 EVT MemVT, MachinePointerInfo PtrInfo, MaybeAlign Alignment = None, 1179 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1180 MachineMemOperand::MOStore, 1181 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) { 1182 // Ensure that codegen never sees alignment 0 1183 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, 1184 Alignment.getValueOr(getEVTAlign(MemVT)), Flags, 1185 Size, AAInfo); 1186 } 1187 1188 LLVM_ATTRIBUTE_DEPRECATED( 1189 inline SDValue getMemIntrinsicNode( 1190 unsigned Opcode, const SDLoc &dl, SDVTList VTList, 1191 ArrayRef<SDValue> Ops, EVT MemVT, MachinePointerInfo PtrInfo, 1192 unsigned Alignment, 1193 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1194 MachineMemOperand::MOStore, 1195 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()), 1196 "") { 1197 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, 1198 MaybeAlign(Alignment), Flags, Size, AAInfo); 1199 } 1200 1201 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, 1202 ArrayRef<SDValue> Ops, EVT MemVT, 1203 MachineMemOperand *MMO); 1204 1205 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends 1206 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between 1207 /// offsets `Offset` and `Offset + Size`. 1208 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, 1209 int FrameIndex, int64_t Size, int64_t Offset = -1); 1210 1211 /// Create a MERGE_VALUES node from the given operands. 1212 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl); 1213 1214 /// Loads are not normal binary operators: their result type is not 1215 /// determined by their operands, and they produce a value AND a token chain. 1216 /// 1217 /// This function will set the MOLoad flag on MMOFlags, but you can set it if 1218 /// you want. The MOStore flag must not be set. 1219 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1220 MachinePointerInfo PtrInfo, MaybeAlign Alignment, 1221 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1222 const AAMDNodes &AAInfo = AAMDNodes(), 1223 const MDNode *Ranges = nullptr); 1224 /// FIXME: Remove once transition to Align is over. 1225 inline SDValue 1226 getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1227 MachinePointerInfo PtrInfo, unsigned Alignment = 0, 1228 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1229 const AAMDNodes &AAInfo = AAMDNodes(), 1230 const MDNode *Ranges = nullptr) { 1231 return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags, 1232 AAInfo, Ranges); 1233 } 1234 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1235 MachineMemOperand *MMO); 1236 SDValue 1237 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, 1238 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, 1239 MaybeAlign Alignment, 1240 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1241 const AAMDNodes &AAInfo = AAMDNodes()); 1242 /// FIXME: Remove once transition to Align is over. 1243 inline SDValue 1244 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, 1245 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, 1246 unsigned Alignment = 0, 1247 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1248 const AAMDNodes &AAInfo = AAMDNodes()) { 1249 return getExtLoad(ExtType, dl, VT, Chain, Ptr, PtrInfo, MemVT, 1250 MaybeAlign(Alignment), MMOFlags, AAInfo); 1251 } 1252 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1253 SDValue Chain, SDValue Ptr, EVT MemVT, 1254 MachineMemOperand *MMO); 1255 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1256 SDValue Offset, ISD::MemIndexedMode AM); 1257 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1258 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1259 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, 1260 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1261 const AAMDNodes &AAInfo = AAMDNodes(), 1262 const MDNode *Ranges = nullptr); 1263 inline SDValue 1264 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1265 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1266 MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, 1267 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1268 const AAMDNodes &AAInfo = AAMDNodes(), 1269 const MDNode *Ranges = nullptr) { 1270 // Ensures that codegen never sees a None Alignment. 1271 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1272 Alignment.getValueOr(getEVTAlign(MemVT)), MMOFlags, AAInfo, 1273 Ranges); 1274 } 1275 /// FIXME: Remove once transition to Align is over. 1276 inline SDValue 1277 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1278 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1279 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment = 0, 1280 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1281 const AAMDNodes &AAInfo = AAMDNodes(), 1282 const MDNode *Ranges = nullptr) { 1283 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1284 MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges); 1285 } 1286 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1287 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1288 EVT MemVT, MachineMemOperand *MMO); 1289 1290 /// Helper function to build ISD::STORE nodes. 1291 /// 1292 /// This function will set the MOStore flag on MMOFlags, but you can set it if 1293 /// you want. The MOLoad and MOInvariant flags must not be set. 1294 1295 SDValue 1296 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1297 MachinePointerInfo PtrInfo, Align Alignment, 1298 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1299 const AAMDNodes &AAInfo = AAMDNodes()); 1300 inline SDValue 1301 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1302 MachinePointerInfo PtrInfo, MaybeAlign Alignment, 1303 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1304 const AAMDNodes &AAInfo = AAMDNodes()) { 1305 return getStore(Chain, dl, Val, Ptr, PtrInfo, 1306 Alignment.getValueOr(getEVTAlign(Val.getValueType())), 1307 MMOFlags, AAInfo); 1308 } 1309 /// FIXME: Remove once transition to Align is over. 1310 inline SDValue 1311 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1312 MachinePointerInfo PtrInfo, unsigned Alignment = 0, 1313 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1314 const AAMDNodes &AAInfo = AAMDNodes()) { 1315 return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment), 1316 MMOFlags, AAInfo); 1317 } 1318 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1319 MachineMemOperand *MMO); 1320 SDValue 1321 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1322 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, 1323 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1324 const AAMDNodes &AAInfo = AAMDNodes()); 1325 inline SDValue 1326 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1327 MachinePointerInfo PtrInfo, EVT SVT, MaybeAlign Alignment, 1328 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1329 const AAMDNodes &AAInfo = AAMDNodes()) { 1330 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1331 Alignment.getValueOr(getEVTAlign(SVT)), MMOFlags, 1332 AAInfo); 1333 } 1334 /// FIXME: Remove once transition to Align is over. 1335 inline SDValue 1336 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1337 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment = 0, 1338 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1339 const AAMDNodes &AAInfo = AAMDNodes()) { 1340 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1341 MaybeAlign(Alignment), MMOFlags, AAInfo); 1342 } 1343 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1344 SDValue Ptr, EVT SVT, MachineMemOperand *MMO); 1345 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, 1346 SDValue Offset, ISD::MemIndexedMode AM); 1347 1348 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, 1349 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, 1350 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1351 ISD::LoadExtType, bool IsExpanding = false); 1352 SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1353 SDValue Offset, ISD::MemIndexedMode AM); 1354 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1355 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, 1356 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1357 bool IsTruncating = false, bool IsCompressing = false); 1358 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, 1359 SDValue Base, SDValue Offset, 1360 ISD::MemIndexedMode AM); 1361 SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, 1362 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1363 ISD::MemIndexType IndexType); 1364 SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, 1365 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1366 ISD::MemIndexType IndexType); 1367 1368 /// Construct a node to track a Value* through the backend. 1369 SDValue getSrcValue(const Value *v); 1370 1371 /// Return an MDNodeSDNode which holds an MDNode. 1372 SDValue getMDNode(const MDNode *MD); 1373 1374 /// Return a bitcast using the SDLoc of the value operand, and casting to the 1375 /// provided type. Use getNode to set a custom SDLoc. 1376 SDValue getBitcast(EVT VT, SDValue V); 1377 1378 /// Return an AddrSpaceCastSDNode. 1379 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, 1380 unsigned DestAS); 1381 1382 /// Return a freeze using the SDLoc of the value operand. 1383 SDValue getFreeze(SDValue V); 1384 1385 /// Return an AssertAlignSDNode. 1386 SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A); 1387 1388 /// Return the specified value casted to 1389 /// the target's desired shift amount type. 1390 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); 1391 1392 /// Expand the specified \c ISD::VAARG node as the Legalize pass would. 1393 SDValue expandVAArg(SDNode *Node); 1394 1395 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. 1396 SDValue expandVACopy(SDNode *Node); 1397 1398 /// Returs an GlobalAddress of the function from the current module with 1399 /// name matching the given ExternalSymbol. Additionally can provide the 1400 /// matched function. 1401 /// Panics the function doesn't exists. 1402 SDValue getSymbolFunctionGlobalAddress(SDValue Op, 1403 Function **TargetFunction = nullptr); 1404 1405 /// *Mutate* the specified node in-place to have the 1406 /// specified operands. If the resultant node already exists in the DAG, 1407 /// this does not modify the specified node, instead it returns the node that 1408 /// already exists. If the resultant node does not exist in the DAG, the 1409 /// input node is returned. As a degenerate case, if you specify the same 1410 /// input operands as the node already has, the input node is returned. 1411 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op); 1412 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2); 1413 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1414 SDValue Op3); 1415 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1416 SDValue Op3, SDValue Op4); 1417 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1418 SDValue Op3, SDValue Op4, SDValue Op5); 1419 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); 1420 1421 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k 1422 /// values or more, move values into new TokenFactors in 64k-1 blocks, until 1423 /// the final TokenFactor has less than 64k operands. 1424 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals); 1425 1426 /// *Mutate* the specified machine node's memory references to the provided 1427 /// list. 1428 void setNodeMemRefs(MachineSDNode *N, 1429 ArrayRef<MachineMemOperand *> NewMemRefs); 1430 1431 // Propagates the change in divergence to users 1432 void updateDivergence(SDNode * N); 1433 1434 /// These are used for target selectors to *mutate* the 1435 /// specified node to have the specified return type, Target opcode, and 1436 /// operands. Note that target opcodes are stored as 1437 /// ~TargetOpcode in the node opcode field. The resultant node is returned. 1438 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT); 1439 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1); 1440 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1441 SDValue Op1, SDValue Op2); 1442 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1443 SDValue Op1, SDValue Op2, SDValue Op3); 1444 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1445 ArrayRef<SDValue> Ops); 1446 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2); 1447 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1448 EVT VT2, ArrayRef<SDValue> Ops); 1449 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1450 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1451 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1, 1452 EVT VT2, SDValue Op1); 1453 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1454 EVT VT2, SDValue Op1, SDValue Op2); 1455 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs, 1456 ArrayRef<SDValue> Ops); 1457 1458 /// This *mutates* the specified node to have the specified 1459 /// return type, opcode, and operands. 1460 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, 1461 ArrayRef<SDValue> Ops); 1462 1463 /// Mutate the specified strict FP node to its non-strict equivalent, 1464 /// unlinking the node from its chain and dropping the metadata arguments. 1465 /// The node must be a strict FP node. 1466 SDNode *mutateStrictFPToFP(SDNode *Node); 1467 1468 /// These are used for target selectors to create a new node 1469 /// with specified return type(s), MachineInstr opcode, and operands. 1470 /// 1471 /// Note that getMachineNode returns the resultant node. If there is already 1472 /// a node of the specified opcode and operands, it returns that node instead 1473 /// of the current one. 1474 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT); 1475 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1476 SDValue Op1); 1477 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1478 SDValue Op1, SDValue Op2); 1479 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1480 SDValue Op1, SDValue Op2, SDValue Op3); 1481 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1482 ArrayRef<SDValue> Ops); 1483 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1484 EVT VT2, SDValue Op1, SDValue Op2); 1485 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1486 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3); 1487 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1488 EVT VT2, ArrayRef<SDValue> Ops); 1489 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1490 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2); 1491 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1492 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2, 1493 SDValue Op3); 1494 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1495 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1496 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, 1497 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); 1498 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs, 1499 ArrayRef<SDValue> Ops); 1500 1501 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes. 1502 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1503 SDValue Operand); 1504 1505 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes. 1506 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1507 SDValue Operand, SDValue Subreg); 1508 1509 /// Get the specified node if it's already available, or else return NULL. 1510 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops, 1511 const SDNodeFlags Flags = SDNodeFlags()); 1512 1513 /// Creates a SDDbgValue node. 1514 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, 1515 unsigned R, bool IsIndirect, const DebugLoc &DL, 1516 unsigned O); 1517 1518 /// Creates a constant SDDbgValue node. 1519 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr, 1520 const Value *C, const DebugLoc &DL, 1521 unsigned O); 1522 1523 /// Creates a FrameIndex SDDbgValue node. 1524 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, 1525 unsigned FI, bool IsIndirect, 1526 const DebugLoc &DL, unsigned O); 1527 1528 /// Creates a VReg SDDbgValue node. 1529 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr, 1530 unsigned VReg, bool IsIndirect, 1531 const DebugLoc &DL, unsigned O); 1532 1533 /// Creates a SDDbgLabel node. 1534 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O); 1535 1536 /// Transfer debug values from one node to another, while optionally 1537 /// generating fragment expressions for split-up values. If \p InvalidateDbg 1538 /// is set, debug values are invalidated after they are transferred. 1539 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0, 1540 unsigned SizeInBits = 0, bool InvalidateDbg = true); 1541 1542 /// Remove the specified node from the system. If any of its 1543 /// operands then becomes dead, remove them as well. Inform UpdateListener 1544 /// for each node deleted. 1545 void RemoveDeadNode(SDNode *N); 1546 1547 /// This method deletes the unreachable nodes in the 1548 /// given list, and any nodes that become unreachable as a result. 1549 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes); 1550 1551 /// Modify anything using 'From' to use 'To' instead. 1552 /// This can cause recursive merging of nodes in the DAG. Use the first 1553 /// version if 'From' is known to have a single result, use the second 1554 /// if you have two nodes with identical results (or if 'To' has a superset 1555 /// of the results of 'From'), use the third otherwise. 1556 /// 1557 /// These methods all take an optional UpdateListener, which (if not null) is 1558 /// informed about nodes that are deleted and modified due to recursive 1559 /// changes in the dag. 1560 /// 1561 /// These functions only replace all existing uses. It's possible that as 1562 /// these replacements are being performed, CSE may cause the From node 1563 /// to be given new uses. These new uses of From are left in place, and 1564 /// not automatically transferred to To. 1565 /// 1566 void ReplaceAllUsesWith(SDValue From, SDValue To); 1567 void ReplaceAllUsesWith(SDNode *From, SDNode *To); 1568 void ReplaceAllUsesWith(SDNode *From, const SDValue *To); 1569 1570 /// Replace any uses of From with To, leaving 1571 /// uses of other values produced by From.getNode() alone. 1572 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); 1573 1574 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. 1575 /// This correctly handles the case where 1576 /// there is an overlap between the From values and the To values. 1577 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, 1578 unsigned Num); 1579 1580 /// If an existing load has uses of its chain, create a token factor node with 1581 /// that chain and the new memory node's chain and update users of the old 1582 /// chain to the token factor. This ensures that the new memory node will have 1583 /// the same relative memory dependency position as the old load. Returns the 1584 /// new merged load chain. 1585 SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New); 1586 1587 /// Topological-sort the AllNodes list and a 1588 /// assign a unique node id for each node in the DAG based on their 1589 /// topological order. Returns the number of nodes. 1590 unsigned AssignTopologicalOrder(); 1591 1592 /// Move node N in the AllNodes list to be immediately 1593 /// before the given iterator Position. This may be used to update the 1594 /// topological ordering when the list of nodes is modified. 1595 void RepositionNode(allnodes_iterator Position, SDNode *N) { 1596 AllNodes.insert(Position, AllNodes.remove(N)); 1597 } 1598 1599 /// Returns an APFloat semantics tag appropriate for the given type. If VT is 1600 /// a vector type, the element semantics are returned. 1601 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) { 1602 switch (VT.getScalarType().getSimpleVT().SimpleTy) { 1603 default: llvm_unreachable("Unknown FP format"); 1604 case MVT::f16: return APFloat::IEEEhalf(); 1605 case MVT::bf16: return APFloat::BFloat(); 1606 case MVT::f32: return APFloat::IEEEsingle(); 1607 case MVT::f64: return APFloat::IEEEdouble(); 1608 case MVT::f80: return APFloat::x87DoubleExtended(); 1609 case MVT::f128: return APFloat::IEEEquad(); 1610 case MVT::ppcf128: return APFloat::PPCDoubleDouble(); 1611 } 1612 } 1613 1614 /// Add a dbg_value SDNode. If SD is non-null that means the 1615 /// value is produced by SD. 1616 void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter); 1617 1618 /// Add a dbg_label SDNode. 1619 void AddDbgLabel(SDDbgLabel *DB); 1620 1621 /// Get the debug values which reference the given SDNode. 1622 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const { 1623 return DbgInfo->getSDDbgValues(SD); 1624 } 1625 1626 public: 1627 /// Return true if there are any SDDbgValue nodes associated 1628 /// with this SelectionDAG. 1629 bool hasDebugValues() const { return !DbgInfo->empty(); } 1630 1631 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); } 1632 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); } 1633 1634 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const { 1635 return DbgInfo->ByvalParmDbgBegin(); 1636 } 1637 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const { 1638 return DbgInfo->ByvalParmDbgEnd(); 1639 } 1640 1641 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const { 1642 return DbgInfo->DbgLabelBegin(); 1643 } 1644 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const { 1645 return DbgInfo->DbgLabelEnd(); 1646 } 1647 1648 /// To be invoked on an SDNode that is slated to be erased. This 1649 /// function mirrors \c llvm::salvageDebugInfo. 1650 void salvageDebugInfo(SDNode &N); 1651 1652 void dump() const; 1653 1654 /// In most cases this function returns the ABI alignment for a given type, 1655 /// except for illegal vector types where the alignment exceeds that of the 1656 /// stack. In such cases we attempt to break the vector down to a legal type 1657 /// and return the ABI alignment for that instead. 1658 Align getReducedAlign(EVT VT, bool UseABI); 1659 1660 /// Create a stack temporary based on the size in bytes and the alignment 1661 SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment); 1662 1663 /// Create a stack temporary, suitable for holding the specified value type. 1664 /// If minAlign is specified, the slot size will have at least that alignment. 1665 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); 1666 1667 /// Create a stack temporary suitable for holding either of the specified 1668 /// value types. 1669 SDValue CreateStackTemporary(EVT VT1, EVT VT2); 1670 1671 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, 1672 const GlobalAddressSDNode *GA, 1673 const SDNode *N2); 1674 1675 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, 1676 ArrayRef<SDValue> Ops); 1677 1678 SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, 1679 ArrayRef<SDValue> Ops, 1680 const SDNodeFlags Flags = SDNodeFlags()); 1681 1682 /// Fold floating-point operations with 2 operands when both operands are 1683 /// constants and/or undefined. 1684 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, 1685 SDValue N1, SDValue N2); 1686 1687 /// Constant fold a setcc to true or false. 1688 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, 1689 const SDLoc &dl); 1690 1691 /// See if the specified operand can be simplified with the knowledge that 1692 /// only the bits specified by DemandedBits are used. If so, return the 1693 /// simpler operand, otherwise return a null SDValue. 1694 /// 1695 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can 1696 /// simplify nodes with multiple uses more aggressively.) 1697 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits); 1698 1699 /// See if the specified operand can be simplified with the knowledge that 1700 /// only the bits specified by DemandedBits are used in the elements specified 1701 /// by DemandedElts. If so, return the simpler operand, otherwise return a 1702 /// null SDValue. 1703 /// 1704 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can 1705 /// simplify nodes with multiple uses more aggressively.) 1706 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits, 1707 const APInt &DemandedElts); 1708 1709 /// Return true if the sign bit of Op is known to be zero. 1710 /// We use this predicate to simplify operations downstream. 1711 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; 1712 1713 /// Return true if 'Op & Mask' is known to be zero. We 1714 /// use this predicate to simplify operations downstream. Op and Mask are 1715 /// known to be the same type. 1716 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1717 unsigned Depth = 0) const; 1718 1719 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We 1720 /// use this predicate to simplify operations downstream. Op and Mask are 1721 /// known to be the same type. 1722 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1723 const APInt &DemandedElts, unsigned Depth = 0) const; 1724 1725 /// Return true if '(Op & Mask) == Mask'. 1726 /// Op and Mask are known to be the same type. 1727 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, 1728 unsigned Depth = 0) const; 1729 1730 /// Determine which bits of Op are known to be either zero or one and return 1731 /// them in Known. For vectors, the known bits are those that are shared by 1732 /// every vector element. 1733 /// Targets can implement the computeKnownBitsForTargetNode method in the 1734 /// TargetLowering class to allow target nodes to be understood. 1735 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; 1736 1737 /// Determine which bits of Op are known to be either zero or one and return 1738 /// them in Known. The DemandedElts argument allows us to only collect the 1739 /// known bits that are shared by the requested vector elements. 1740 /// Targets can implement the computeKnownBitsForTargetNode method in the 1741 /// TargetLowering class to allow target nodes to be understood. 1742 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, 1743 unsigned Depth = 0) const; 1744 1745 /// Used to represent the possible overflow behavior of an operation. 1746 /// Never: the operation cannot overflow. 1747 /// Always: the operation will always overflow. 1748 /// Sometime: the operation may or may not overflow. 1749 enum OverflowKind { 1750 OFK_Never, 1751 OFK_Sometime, 1752 OFK_Always, 1753 }; 1754 1755 /// Determine if the result of the addition of 2 node can overflow. 1756 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const; 1757 1758 /// Test if the given value is known to have exactly one bit set. This differs 1759 /// from computeKnownBits in that it doesn't necessarily determine which bit 1760 /// is set. 1761 bool isKnownToBeAPowerOfTwo(SDValue Val) const; 1762 1763 /// Return the number of times the sign bit of the register is replicated into 1764 /// the other bits. We know that at least 1 bit is always equal to the sign 1765 /// bit (itself), but other cases can give us information. For example, 1766 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 1767 /// to each other, so we return 3. Targets can implement the 1768 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow 1769 /// target nodes to be understood. 1770 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const; 1771 1772 /// Return the number of times the sign bit of the register is replicated into 1773 /// the other bits. We know that at least 1 bit is always equal to the sign 1774 /// bit (itself), but other cases can give us information. For example, 1775 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 1776 /// to each other, so we return 3. The DemandedElts argument allows 1777 /// us to only collect the minimum sign bits of the requested vector elements. 1778 /// Targets can implement the ComputeNumSignBitsForTarget method in the 1779 /// TargetLowering class to allow target nodes to be understood. 1780 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 1781 unsigned Depth = 0) const; 1782 1783 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode 1784 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that 1785 /// is guaranteed to have the same semantics as an ADD. This handles the 1786 /// equivalence: 1787 /// X|Cst == X+Cst iff X&Cst = 0. 1788 bool isBaseWithConstantOffset(SDValue Op) const; 1789 1790 /// Test whether the given SDValue is known to never be NaN. If \p SNaN is 1791 /// true, returns if \p Op is known to never be a signaling NaN (it may still 1792 /// be a qNaN). 1793 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; 1794 1795 /// \returns true if \p Op is known to never be a signaling NaN. 1796 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { 1797 return isKnownNeverNaN(Op, true, Depth); 1798 } 1799 1800 /// Test whether the given floating point SDValue is known to never be 1801 /// positive or negative zero. 1802 bool isKnownNeverZeroFloat(SDValue Op) const; 1803 1804 /// Test whether the given SDValue is known to contain non-zero value(s). 1805 bool isKnownNeverZero(SDValue Op) const; 1806 1807 /// Test whether two SDValues are known to compare equal. This 1808 /// is true if they are the same value, or if one is negative zero and the 1809 /// other positive zero. 1810 bool isEqualTo(SDValue A, SDValue B) const; 1811 1812 /// Return true if A and B have no common bits set. As an example, this can 1813 /// allow an 'add' to be transformed into an 'or'. 1814 bool haveNoCommonBitsSet(SDValue A, SDValue B) const; 1815 1816 /// Test whether \p V has a splatted value for all the demanded elements. 1817 /// 1818 /// On success \p UndefElts will indicate the elements that have UNDEF 1819 /// values instead of the splat value, this is only guaranteed to be correct 1820 /// for \p DemandedElts. 1821 /// 1822 /// NOTE: The function will return true for a demanded splat of UNDEF values. 1823 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts); 1824 1825 /// Test whether \p V has a splatted value. 1826 bool isSplatValue(SDValue V, bool AllowUndefs = false); 1827 1828 /// If V is a splatted value, return the source vector and its splat index. 1829 SDValue getSplatSourceVector(SDValue V, int &SplatIndex); 1830 1831 /// If V is a splat vector, return its scalar source operand by extracting 1832 /// that element from the source vector. 1833 SDValue getSplatValue(SDValue V); 1834 1835 /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount 1836 /// that is less than the element bit-width of the shift node, return it. 1837 const APInt *getValidShiftAmountConstant(SDValue V, 1838 const APInt &DemandedElts) const; 1839 1840 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 1841 /// than the element bit-width of the shift node, return the minimum value. 1842 const APInt * 1843 getValidMinimumShiftAmountConstant(SDValue V, 1844 const APInt &DemandedElts) const; 1845 1846 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 1847 /// than the element bit-width of the shift node, return the maximum value. 1848 const APInt * 1849 getValidMaximumShiftAmountConstant(SDValue V, 1850 const APInt &DemandedElts) const; 1851 1852 /// Match a binop + shuffle pyramid that represents a horizontal reduction 1853 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p 1854 /// Extract. The reduction must use one of the opcodes listed in /p 1855 /// CandidateBinOps and on success /p BinOp will contain the matching opcode. 1856 /// Returns the vector that is being reduced on, or SDValue() if a reduction 1857 /// was not matched. If \p AllowPartials is set then in the case of a 1858 /// reduction pattern that only matches the first few stages, the extracted 1859 /// subvector of the start of the reduction is returned. 1860 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, 1861 ArrayRef<ISD::NodeType> CandidateBinOps, 1862 bool AllowPartials = false); 1863 1864 /// Utility function used by legalize and lowering to 1865 /// "unroll" a vector operation by splitting out the scalars and operating 1866 /// on each element individually. If the ResNE is 0, fully unroll the vector 1867 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. 1868 /// If the ResNE is greater than the width of the vector op, unroll the 1869 /// vector op and fill the end of the resulting vector with UNDEFS. 1870 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); 1871 1872 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes. 1873 /// This is a separate function because those opcodes have two results. 1874 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N, 1875 unsigned ResNE = 0); 1876 1877 /// Return true if loads are next to each other and can be 1878 /// merged. Check that both are nonvolatile and if LD is loading 1879 /// 'Bytes' bytes from a location that is 'Dist' units away from the 1880 /// location that the 'Base' load is loading from. 1881 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, 1882 unsigned Bytes, int Dist) const; 1883 1884 /// Infer alignment of a load / store address. Return None if it cannot be 1885 /// inferred. 1886 MaybeAlign InferPtrAlign(SDValue Ptr) const; 1887 1888 LLVM_ATTRIBUTE_DEPRECATED(inline unsigned InferPtrAlignment(SDValue Ptr) 1889 const, 1890 "Use InferPtrAlign instead") { 1891 if (auto A = InferPtrAlign(Ptr)) 1892 return A->value(); 1893 return 0; 1894 } 1895 1896 /// Compute the VTs needed for the low/hi parts of a type 1897 /// which is split (or expanded) into two not necessarily identical pieces. 1898 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const; 1899 1900 /// Compute the VTs needed for the low/hi parts of a type, dependent on an 1901 /// enveloping VT that has been split into two identical pieces. Sets the 1902 /// HisIsEmpty flag when hi type has zero storage size. 1903 std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, 1904 bool *HiIsEmpty) const; 1905 1906 /// Split the vector with EXTRACT_SUBVECTOR using the provides 1907 /// VTs and return the low/high part. 1908 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL, 1909 const EVT &LoVT, const EVT &HiVT); 1910 1911 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part. 1912 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { 1913 EVT LoVT, HiVT; 1914 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); 1915 return SplitVector(N, DL, LoVT, HiVT); 1916 } 1917 1918 /// Split the node's operand with EXTRACT_SUBVECTOR and 1919 /// return the low/high part. 1920 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) 1921 { 1922 return SplitVector(N->getOperand(OpNo), SDLoc(N)); 1923 } 1924 1925 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 1926 SDValue WidenVector(const SDValue &N, const SDLoc &DL); 1927 1928 /// Append the extracted elements from Start to Count out of the vector Op in 1929 /// Args. If Count is 0, all of the elements will be extracted. The extracted 1930 /// elements will have type EVT if it is provided, and otherwise their type 1931 /// will be Op's element type. 1932 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, 1933 unsigned Start = 0, unsigned Count = 0, 1934 EVT EltVT = EVT()); 1935 1936 /// Compute the default alignment value for the given type. 1937 Align getEVTAlign(EVT MemoryVT) const; 1938 /// Compute the default alignment value for the given type. 1939 /// FIXME: Remove once transition to Align is over. 1940 inline unsigned getEVTAlignment(EVT MemoryVT) const { 1941 return getEVTAlign(MemoryVT).value(); 1942 } 1943 1944 /// Test whether the given value is a constant int or similar node. 1945 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N); 1946 1947 /// Test whether the given value is a constant FP or similar node. 1948 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N); 1949 1950 /// \returns true if \p N is any kind of constant or build_vector of 1951 /// constants, int or float. If a vector, it may not necessarily be a splat. 1952 inline bool isConstantValueOfAnyType(SDValue N) { 1953 return isConstantIntBuildVectorOrConstantInt(N) || 1954 isConstantFPBuildVectorOrConstantFP(N); 1955 } 1956 1957 void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { 1958 SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo); 1959 } 1960 1961 CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) { 1962 auto I = SDCallSiteDbgInfo.find(CallNode); 1963 if (I != SDCallSiteDbgInfo.end()) 1964 return std::move(I->second).CSInfo; 1965 return CallSiteInfo(); 1966 } 1967 1968 void addHeapAllocSite(const SDNode *Node, MDNode *MD) { 1969 SDCallSiteDbgInfo[Node].HeapAllocSite = MD; 1970 } 1971 1972 /// Return the HeapAllocSite type associated with the SDNode, if it exists. 1973 MDNode *getHeapAllocSite(const SDNode *Node) { 1974 auto It = SDCallSiteDbgInfo.find(Node); 1975 if (It == SDCallSiteDbgInfo.end()) 1976 return nullptr; 1977 return It->second.HeapAllocSite; 1978 } 1979 1980 void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { 1981 if (NoMerge) 1982 SDCallSiteDbgInfo[Node].NoMerge = NoMerge; 1983 } 1984 1985 bool getNoMergeSiteInfo(const SDNode *Node) { 1986 auto I = SDCallSiteDbgInfo.find(Node); 1987 if (I == SDCallSiteDbgInfo.end()) 1988 return false; 1989 return I->second.NoMerge; 1990 } 1991 1992 /// Return the current function's default denormal handling kind for the given 1993 /// floating point type. 1994 DenormalMode getDenormalMode(EVT VT) const { 1995 return MF->getDenormalMode(EVTToAPFloatSemantics(VT)); 1996 } 1997 1998 bool shouldOptForSize() const; 1999 2000 private: 2001 void InsertNode(SDNode *N); 2002 bool RemoveNodeFromCSEMaps(SDNode *N); 2003 void AddModifiedNodeToCSEMaps(SDNode *N); 2004 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos); 2005 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2, 2006 void *&InsertPos); 2007 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 2008 void *&InsertPos); 2009 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc); 2010 2011 void DeleteNodeNotInCSEMaps(SDNode *N); 2012 void DeallocateNode(SDNode *N); 2013 2014 void allnodes_clear(); 2015 2016 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2017 /// not, return the insertion token that will make insertion faster. This 2018 /// overload is for nodes other than Constant or ConstantFP, use the other one 2019 /// for those. 2020 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos); 2021 2022 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2023 /// not, return the insertion token that will make insertion faster. Performs 2024 /// additional processing for constant nodes. 2025 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL, 2026 void *&InsertPos); 2027 2028 /// List of non-single value types. 2029 FoldingSet<SDVTListNode> VTListMap; 2030 2031 /// Maps to auto-CSE operations. 2032 std::vector<CondCodeSDNode*> CondCodeNodes; 2033 2034 std::vector<SDNode*> ValueTypeNodes; 2035 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes; 2036 StringMap<SDNode*> ExternalSymbols; 2037 2038 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols; 2039 DenseMap<MCSymbol *, SDNode *> MCSymbols; 2040 }; 2041 2042 template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> { 2043 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>; 2044 2045 static nodes_iterator nodes_begin(SelectionDAG *G) { 2046 return nodes_iterator(G->allnodes_begin()); 2047 } 2048 2049 static nodes_iterator nodes_end(SelectionDAG *G) { 2050 return nodes_iterator(G->allnodes_end()); 2051 } 2052 }; 2053 2054 } // end namespace llvm 2055 2056 #endif // LLVM_CODEGEN_SELECTIONDAG_H 2057