10b57cec5SDimitry Andric //===- VPlanValue.h - Represent Values in Vectorizer Plan -----------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric /// 90b57cec5SDimitry Andric /// \file 100b57cec5SDimitry Andric /// This file contains the declarations of the entities induced by Vectorization 110b57cec5SDimitry Andric /// Plans, e.g. the instructions the VPlan intends to generate if executed. 120b57cec5SDimitry Andric /// VPlan models the following entities: 13e8d8bef9SDimitry Andric /// VPValue VPUser VPDef 14e8d8bef9SDimitry Andric /// | | 15e8d8bef9SDimitry Andric /// VPInstruction 160b57cec5SDimitry Andric /// These are documented in docs/VectorizationPlan.rst. 170b57cec5SDimitry Andric /// 180b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 190b57cec5SDimitry Andric 200b57cec5SDimitry Andric #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H 210b57cec5SDimitry Andric #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H 220b57cec5SDimitry Andric 230b57cec5SDimitry Andric #include "llvm/ADT/DenseMap.h" 24e8d8bef9SDimitry Andric #include "llvm/ADT/STLExtras.h" 250b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h" 26e8d8bef9SDimitry Andric #include "llvm/ADT/TinyPtrVector.h" 275ffd83dbSDimitry Andric #include "llvm/ADT/iterator_range.h" 280b57cec5SDimitry Andric 290b57cec5SDimitry Andric namespace llvm { 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric // Forward declarations. 325ffd83dbSDimitry Andric class raw_ostream; 335ffd83dbSDimitry Andric class Value; 34e8d8bef9SDimitry Andric class VPDef; 355ffd83dbSDimitry Andric class VPSlotTracker; 360b57cec5SDimitry Andric class VPUser; 37e8d8bef9SDimitry Andric class VPRecipeBase; 38e8d8bef9SDimitry Andric class VPWidenMemoryInstructionRecipe; 390b57cec5SDimitry Andric 400b57cec5SDimitry Andric // This is the base class of the VPlan Def/Use graph, used for modeling the data 410b57cec5SDimitry Andric // flow into, within and out of the VPlan. VPValues can stand for live-ins 420b57cec5SDimitry Andric // coming from the input IR, instructions which VPlan will generate if executed 430b57cec5SDimitry Andric // and live-outs which the VPlan will need to fix accordingly. 440b57cec5SDimitry Andric class VPValue { 450b57cec5SDimitry Andric friend class VPBuilder; 46e8d8bef9SDimitry Andric friend class VPDef; 47e8d8bef9SDimitry Andric friend class VPInstruction; 485ffd83dbSDimitry Andric friend struct VPlanTransforms; 490b57cec5SDimitry Andric friend class VPBasicBlock; 500b57cec5SDimitry Andric friend class VPInterleavedAccessInfo; 515ffd83dbSDimitry Andric friend class VPSlotTracker; 52e8d8bef9SDimitry Andric friend class VPRecipeBase; 53e8d8bef9SDimitry Andric friend class VPWidenMemoryInstructionRecipe; 540b57cec5SDimitry Andric 550b57cec5SDimitry Andric const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). 560b57cec5SDimitry Andric 570b57cec5SDimitry Andric SmallVector<VPUser *, 1> Users; 580b57cec5SDimitry Andric 590b57cec5SDimitry Andric protected: 600b57cec5SDimitry Andric // Hold the underlying Value, if any, attached to this VPValue. 610b57cec5SDimitry Andric Value *UnderlyingVal; 620b57cec5SDimitry Andric 63e8d8bef9SDimitry Andric /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the 64e8d8bef9SDimitry Andric /// VPValue is not defined by any recipe modeled in VPlan. 65e8d8bef9SDimitry Andric VPDef *Def; 66e8d8bef9SDimitry Andric 67e8d8bef9SDimitry Andric VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr); 680b57cec5SDimitry Andric 690b57cec5SDimitry Andric // DESIGN PRINCIPLE: Access to the underlying IR must be strictly limited to 700b57cec5SDimitry Andric // the front-end and back-end of VPlan so that the middle-end is as 710b57cec5SDimitry Andric // independent as possible of the underlying IR. We grant access to the 720b57cec5SDimitry Andric // underlying IR using friendship. In that way, we should be able to use VPlan 730b57cec5SDimitry Andric // for multiple underlying IRs (Polly?) by providing a new VPlan front-end, 740b57cec5SDimitry Andric // back-end and analysis information for the new IR. 750b57cec5SDimitry Andric 760b57cec5SDimitry Andric // Set \p Val as the underlying Value of this VPValue. setUnderlyingValue(Value * Val)770b57cec5SDimitry Andric void setUnderlyingValue(Value *Val) { 780b57cec5SDimitry Andric assert(!UnderlyingVal && "Underlying Value is already set."); 790b57cec5SDimitry Andric UnderlyingVal = Val; 800b57cec5SDimitry Andric } 810b57cec5SDimitry Andric 820b57cec5SDimitry Andric public: 83e8d8bef9SDimitry Andric /// Return the underlying Value attached to this VPValue. getUnderlyingValue()84e8d8bef9SDimitry Andric Value *getUnderlyingValue() { return UnderlyingVal; } getUnderlyingValue()85e8d8bef9SDimitry Andric const Value *getUnderlyingValue() const { return UnderlyingVal; } 86e8d8bef9SDimitry Andric 870b57cec5SDimitry Andric /// An enumeration for keeping track of the concrete subclass of VPValue that 88bdd1243dSDimitry Andric /// are actually instantiated. 89e8d8bef9SDimitry Andric enum { 90bdd1243dSDimitry Andric VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe 91bdd1243dSDimitry Andric /// that defines multiple values. 92bdd1243dSDimitry Andric VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase. 93e8d8bef9SDimitry Andric }; 940b57cec5SDimitry Andric 95bdd1243dSDimitry Andric /// Create a live-in VPValue. VPValue(VPValueSC,UV,nullptr)96bdd1243dSDimitry Andric VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {} 97bdd1243dSDimitry Andric /// Create a VPValue for a \p Def which is a subclass of VPValue. VPValue(VPVRecipeSC,UV,Def)98bdd1243dSDimitry Andric VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} 99bdd1243dSDimitry Andric /// Create a VPValue for a \p Def which defines multiple values. VPValue(Value * UV,VPDef * Def)100bdd1243dSDimitry Andric VPValue(Value *UV, VPDef *Def) : VPValue(VPValueSC, UV, Def) {} 1010b57cec5SDimitry Andric VPValue(const VPValue &) = delete; 1020b57cec5SDimitry Andric VPValue &operator=(const VPValue &) = delete; 1030b57cec5SDimitry Andric 104e8d8bef9SDimitry Andric virtual ~VPValue(); 105e8d8bef9SDimitry Andric 1060b57cec5SDimitry Andric /// \return an ID for the concrete type of this object. 1070b57cec5SDimitry Andric /// This is used to implement the classof checks. This should not be used 1080b57cec5SDimitry Andric /// for any other purpose, as the values may change as LLVM evolves. getVPValueID()1090b57cec5SDimitry Andric unsigned getVPValueID() const { return SubclassID; } 1100b57cec5SDimitry Andric 111fe6060f1SDimitry Andric #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1125ffd83dbSDimitry Andric void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const; 1135ffd83dbSDimitry Andric void print(raw_ostream &OS, VPSlotTracker &Tracker) const; 1140b57cec5SDimitry Andric 115e8d8bef9SDimitry Andric /// Dump the value to stderr (for debugging). 116e8d8bef9SDimitry Andric void dump() const; 117fe6060f1SDimitry Andric #endif 118e8d8bef9SDimitry Andric getNumUsers()1190b57cec5SDimitry Andric unsigned getNumUsers() const { return Users.size(); } addUser(VPUser & User)1200b57cec5SDimitry Andric void addUser(VPUser &User) { Users.push_back(&User); } 1210b57cec5SDimitry Andric 122e8d8bef9SDimitry Andric /// Remove a single \p User from the list of users. removeUser(VPUser & User)123e8d8bef9SDimitry Andric void removeUser(VPUser &User) { 124e8d8bef9SDimitry Andric // The same user can be added multiple times, e.g. because the same VPValue 125e8d8bef9SDimitry Andric // is used twice by the same VPUser. Remove a single one. 1265f757f3fSDimitry Andric auto *I = find(Users, &User); 1275f757f3fSDimitry Andric if (I != Users.end()) 1285f757f3fSDimitry Andric Users.erase(I); 129e8d8bef9SDimitry Andric } 130e8d8bef9SDimitry Andric 1310b57cec5SDimitry Andric typedef SmallVectorImpl<VPUser *>::iterator user_iterator; 1320b57cec5SDimitry Andric typedef SmallVectorImpl<VPUser *>::const_iterator const_user_iterator; 1330b57cec5SDimitry Andric typedef iterator_range<user_iterator> user_range; 1340b57cec5SDimitry Andric typedef iterator_range<const_user_iterator> const_user_range; 1350b57cec5SDimitry Andric user_begin()1360b57cec5SDimitry Andric user_iterator user_begin() { return Users.begin(); } user_begin()1370b57cec5SDimitry Andric const_user_iterator user_begin() const { return Users.begin(); } user_end()1380b57cec5SDimitry Andric user_iterator user_end() { return Users.end(); } user_end()1390b57cec5SDimitry Andric const_user_iterator user_end() const { return Users.end(); } users()1400b57cec5SDimitry Andric user_range users() { return user_range(user_begin(), user_end()); } users()1410b57cec5SDimitry Andric const_user_range users() const { 1420b57cec5SDimitry Andric return const_user_range(user_begin(), user_end()); 1430b57cec5SDimitry Andric } 1440b57cec5SDimitry Andric 1450b57cec5SDimitry Andric /// Returns true if the value has more than one unique user. hasMoreThanOneUniqueUser()1460b57cec5SDimitry Andric bool hasMoreThanOneUniqueUser() { 1470b57cec5SDimitry Andric if (getNumUsers() == 0) 1480b57cec5SDimitry Andric return false; 1490b57cec5SDimitry Andric 1500b57cec5SDimitry Andric // Check if all users match the first user. 1510b57cec5SDimitry Andric auto Current = std::next(user_begin()); 1520b57cec5SDimitry Andric while (Current != user_end() && *user_begin() == *Current) 1530b57cec5SDimitry Andric Current++; 1540b57cec5SDimitry Andric return Current != user_end(); 1550b57cec5SDimitry Andric } 1560b57cec5SDimitry Andric 1570b57cec5SDimitry Andric void replaceAllUsesWith(VPValue *New); 158e8d8bef9SDimitry Andric 1595f757f3fSDimitry Andric /// Go through the uses list for this VPValue and make each use point to \p 1605f757f3fSDimitry Andric /// New if the callback ShouldReplace returns true for the given use specified 1615f757f3fSDimitry Andric /// by a pair of (VPUser, the use index). 1625f757f3fSDimitry Andric void replaceUsesWithIf( 1635f757f3fSDimitry Andric VPValue *New, 1645f757f3fSDimitry Andric llvm::function_ref<bool(VPUser &U, unsigned Idx)> ShouldReplace); 1655f757f3fSDimitry Andric 166bdd1243dSDimitry Andric /// Returns the recipe defining this VPValue or nullptr if it is not defined 167bdd1243dSDimitry Andric /// by a recipe, i.e. is a live-in. 168bdd1243dSDimitry Andric VPRecipeBase *getDefiningRecipe(); 169bdd1243dSDimitry Andric const VPRecipeBase *getDefiningRecipe() const; 170bdd1243dSDimitry Andric 171bdd1243dSDimitry Andric /// Returns true if this VPValue is defined by a recipe. hasDefiningRecipe()172bdd1243dSDimitry Andric bool hasDefiningRecipe() const { return getDefiningRecipe(); } 173e8d8bef9SDimitry Andric 17406c3fb27SDimitry Andric /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan. isLiveIn()17506c3fb27SDimitry Andric bool isLiveIn() const { return !hasDefiningRecipe(); } 17606c3fb27SDimitry Andric 177e8d8bef9SDimitry Andric /// Returns the underlying IR value, if this VPValue is defined outside the 178e8d8bef9SDimitry Andric /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef 179e8d8bef9SDimitry Andric /// inside a VPlan. getLiveInIRValue()180e8d8bef9SDimitry Andric Value *getLiveInIRValue() { 18106c3fb27SDimitry Andric assert(isLiveIn() && 182e8d8bef9SDimitry Andric "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); 183e8d8bef9SDimitry Andric return getUnderlyingValue(); 184e8d8bef9SDimitry Andric } getLiveInIRValue()18504eeddc0SDimitry Andric const Value *getLiveInIRValue() const { 18606c3fb27SDimitry Andric assert(isLiveIn() && 18704eeddc0SDimitry Andric "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); 18804eeddc0SDimitry Andric return getUnderlyingValue(); 18904eeddc0SDimitry Andric } 190bdd1243dSDimitry Andric 191bdd1243dSDimitry Andric /// Returns true if the VPValue is defined outside any vector regions, i.e. it 192bdd1243dSDimitry Andric /// is a live-in value. 193bdd1243dSDimitry Andric /// TODO: Also handle recipes defined in pre-header blocks. isDefinedOutsideVectorRegions()194bdd1243dSDimitry Andric bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); } 1950b57cec5SDimitry Andric }; 1960b57cec5SDimitry Andric 1970b57cec5SDimitry Andric typedef DenseMap<Value *, VPValue *> Value2VPValueTy; 1980b57cec5SDimitry Andric typedef DenseMap<VPValue *, Value *> VPValue2ValueTy; 1990b57cec5SDimitry Andric 2000b57cec5SDimitry Andric raw_ostream &operator<<(raw_ostream &OS, const VPValue &V); 2010b57cec5SDimitry Andric 2020b57cec5SDimitry Andric /// This class augments VPValue with operands which provide the inverse def-use 2030b57cec5SDimitry Andric /// edges from VPValue's users to their defs. 204e8d8bef9SDimitry Andric class VPUser { 205fe6060f1SDimitry Andric public: 206fe6060f1SDimitry Andric /// Subclass identifier (for isa/dyn_cast). 207fe6060f1SDimitry Andric enum class VPUserID { 208fe6060f1SDimitry Andric Recipe, 20981ad6265SDimitry Andric LiveOut, 210fe6060f1SDimitry Andric }; 211fe6060f1SDimitry Andric 212fe6060f1SDimitry Andric private: 2130b57cec5SDimitry Andric SmallVector<VPValue *, 2> Operands; 2140b57cec5SDimitry Andric 215fe6060f1SDimitry Andric VPUserID ID; 216fe6060f1SDimitry Andric 2170b57cec5SDimitry Andric protected: 218fe6060f1SDimitry Andric #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 219e8d8bef9SDimitry Andric /// Print the operands to \p O. 220e8d8bef9SDimitry Andric void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const; 221fe6060f1SDimitry Andric #endif 222fe6060f1SDimitry Andric VPUser(ArrayRef<VPValue * > Operands,VPUserID ID)223fe6060f1SDimitry Andric VPUser(ArrayRef<VPValue *> Operands, VPUserID ID) : ID(ID) { 224fe6060f1SDimitry Andric for (VPValue *Operand : Operands) 225fe6060f1SDimitry Andric addOperand(Operand); 226fe6060f1SDimitry Andric } 227fe6060f1SDimitry Andric VPUser(std::initializer_list<VPValue * > Operands,VPUserID ID)228fe6060f1SDimitry Andric VPUser(std::initializer_list<VPValue *> Operands, VPUserID ID) 229fe6060f1SDimitry Andric : VPUser(ArrayRef<VPValue *>(Operands), ID) {} 230fe6060f1SDimitry Andric 231fe6060f1SDimitry Andric template <typename IterT> VPUser(iterator_range<IterT> Operands,VPUserID ID)232fe6060f1SDimitry Andric VPUser(iterator_range<IterT> Operands, VPUserID ID) : ID(ID) { 233fe6060f1SDimitry Andric for (VPValue *Operand : Operands) 234fe6060f1SDimitry Andric addOperand(Operand); 235fe6060f1SDimitry Andric } 236e8d8bef9SDimitry Andric 237e8d8bef9SDimitry Andric public: 238fe6060f1SDimitry Andric VPUser() = delete; 2390b57cec5SDimitry Andric VPUser(const VPUser &) = delete; 2400b57cec5SDimitry Andric VPUser &operator=(const VPUser &) = delete; ~VPUser()241e8d8bef9SDimitry Andric virtual ~VPUser() { 242e8d8bef9SDimitry Andric for (VPValue *Op : operands()) 243e8d8bef9SDimitry Andric Op->removeUser(*this); 2440b57cec5SDimitry Andric } 2450b57cec5SDimitry Andric getVPUserID()246fe6060f1SDimitry Andric VPUserID getVPUserID() const { return ID; } 247fe6060f1SDimitry Andric addOperand(VPValue * Operand)2480b57cec5SDimitry Andric void addOperand(VPValue *Operand) { 2490b57cec5SDimitry Andric Operands.push_back(Operand); 2500b57cec5SDimitry Andric Operand->addUser(*this); 2510b57cec5SDimitry Andric } 2520b57cec5SDimitry Andric getNumOperands()2530b57cec5SDimitry Andric unsigned getNumOperands() const { return Operands.size(); } getOperand(unsigned N)2540b57cec5SDimitry Andric inline VPValue *getOperand(unsigned N) const { 2550b57cec5SDimitry Andric assert(N < Operands.size() && "Operand index out of bounds"); 2560b57cec5SDimitry Andric return Operands[N]; 2570b57cec5SDimitry Andric } 2580b57cec5SDimitry Andric setOperand(unsigned I,VPValue * New)259e8d8bef9SDimitry Andric void setOperand(unsigned I, VPValue *New) { 260e8d8bef9SDimitry Andric Operands[I]->removeUser(*this); 261e8d8bef9SDimitry Andric Operands[I] = New; 262e8d8bef9SDimitry Andric New->addUser(*this); 263e8d8bef9SDimitry Andric } 2640b57cec5SDimitry Andric removeLastOperand()265fe6060f1SDimitry Andric void removeLastOperand() { 266fe6060f1SDimitry Andric VPValue *Op = Operands.pop_back_val(); 267fe6060f1SDimitry Andric Op->removeUser(*this); 268fe6060f1SDimitry Andric } 269fe6060f1SDimitry Andric 2700b57cec5SDimitry Andric typedef SmallVectorImpl<VPValue *>::iterator operand_iterator; 2710b57cec5SDimitry Andric typedef SmallVectorImpl<VPValue *>::const_iterator const_operand_iterator; 2720b57cec5SDimitry Andric typedef iterator_range<operand_iterator> operand_range; 2730b57cec5SDimitry Andric typedef iterator_range<const_operand_iterator> const_operand_range; 2740b57cec5SDimitry Andric op_begin()2750b57cec5SDimitry Andric operand_iterator op_begin() { return Operands.begin(); } op_begin()2760b57cec5SDimitry Andric const_operand_iterator op_begin() const { return Operands.begin(); } op_end()2770b57cec5SDimitry Andric operand_iterator op_end() { return Operands.end(); } op_end()2780b57cec5SDimitry Andric const_operand_iterator op_end() const { return Operands.end(); } operands()2790b57cec5SDimitry Andric operand_range operands() { return operand_range(op_begin(), op_end()); } operands()2800b57cec5SDimitry Andric const_operand_range operands() const { 2810b57cec5SDimitry Andric return const_operand_range(op_begin(), op_end()); 2820b57cec5SDimitry Andric } 283e8d8bef9SDimitry Andric 28481ad6265SDimitry Andric /// Returns true if the VPUser uses scalars of operand \p Op. Conservatively 28581ad6265SDimitry Andric /// returns if only first (scalar) lane is used, as default. usesScalars(const VPValue * Op)28681ad6265SDimitry Andric virtual bool usesScalars(const VPValue *Op) const { 28781ad6265SDimitry Andric assert(is_contained(operands(), Op) && 28881ad6265SDimitry Andric "Op must be an operand of the recipe"); 28981ad6265SDimitry Andric return onlyFirstLaneUsed(Op); 29081ad6265SDimitry Andric } 29181ad6265SDimitry Andric 29281ad6265SDimitry Andric /// Returns true if the VPUser only uses the first lane of operand \p Op. 29381ad6265SDimitry Andric /// Conservatively returns false. onlyFirstLaneUsed(const VPValue * Op)29481ad6265SDimitry Andric virtual bool onlyFirstLaneUsed(const VPValue *Op) const { 29581ad6265SDimitry Andric assert(is_contained(operands(), Op) && 29681ad6265SDimitry Andric "Op must be an operand of the recipe"); 29781ad6265SDimitry Andric return false; 29881ad6265SDimitry Andric } 2995f757f3fSDimitry Andric 3005f757f3fSDimitry Andric /// Returns true if the VPUser only uses the first part of operand \p Op. 3015f757f3fSDimitry Andric /// Conservatively returns false. onlyFirstPartUsed(const VPValue * Op)3025f757f3fSDimitry Andric virtual bool onlyFirstPartUsed(const VPValue *Op) const { 3035f757f3fSDimitry Andric assert(is_contained(operands(), Op) && 3045f757f3fSDimitry Andric "Op must be an operand of the recipe"); 3055f757f3fSDimitry Andric return false; 3065f757f3fSDimitry Andric } 3070b57cec5SDimitry Andric }; 308e8d8bef9SDimitry Andric 309e8d8bef9SDimitry Andric /// This class augments a recipe with a set of VPValues defined by the recipe. 310e8d8bef9SDimitry Andric /// It allows recipes to define zero, one or multiple VPValues. A VPDef owns 311e8d8bef9SDimitry Andric /// the VPValues it defines and is responsible for deleting its defined values. 312e8d8bef9SDimitry Andric /// Single-value VPDefs that also inherit from VPValue must make sure to inherit 313e8d8bef9SDimitry Andric /// from VPDef before VPValue. 314e8d8bef9SDimitry Andric class VPDef { 315e8d8bef9SDimitry Andric friend class VPValue; 316e8d8bef9SDimitry Andric 317e8d8bef9SDimitry Andric /// Subclass identifier (for isa/dyn_cast). 318e8d8bef9SDimitry Andric const unsigned char SubclassID; 319e8d8bef9SDimitry Andric 320e8d8bef9SDimitry Andric /// The VPValues defined by this VPDef. 321e8d8bef9SDimitry Andric TinyPtrVector<VPValue *> DefinedValues; 322e8d8bef9SDimitry Andric 323e8d8bef9SDimitry Andric /// Add \p V as a defined value by this VPDef. addDefinedValue(VPValue * V)324e8d8bef9SDimitry Andric void addDefinedValue(VPValue *V) { 325bdd1243dSDimitry Andric assert(V->Def == this && 326e8d8bef9SDimitry Andric "can only add VPValue already linked with this VPDef"); 327e8d8bef9SDimitry Andric DefinedValues.push_back(V); 328e8d8bef9SDimitry Andric } 329e8d8bef9SDimitry Andric 330e8d8bef9SDimitry Andric /// Remove \p V from the values defined by this VPDef. \p V must be a defined 331e8d8bef9SDimitry Andric /// value of this VPDef. removeDefinedValue(VPValue * V)332e8d8bef9SDimitry Andric void removeDefinedValue(VPValue *V) { 333bdd1243dSDimitry Andric assert(V->Def == this && "can only remove VPValue linked with this VPDef"); 334e8d8bef9SDimitry Andric assert(is_contained(DefinedValues, V) && 335e8d8bef9SDimitry Andric "VPValue to remove must be in DefinedValues"); 3365f757f3fSDimitry Andric llvm::erase(DefinedValues, V); 337e8d8bef9SDimitry Andric V->Def = nullptr; 338e8d8bef9SDimitry Andric } 339e8d8bef9SDimitry Andric 340e8d8bef9SDimitry Andric public: 341e8d8bef9SDimitry Andric /// An enumeration for keeping track of the concrete subclass of VPRecipeBase 342e8d8bef9SDimitry Andric /// that is actually instantiated. Values of this enumeration are kept in the 343e8d8bef9SDimitry Andric /// SubclassID field of the VPRecipeBase objects. They are used for concrete 344e8d8bef9SDimitry Andric /// type identification. 345e8d8bef9SDimitry Andric using VPRecipeTy = enum { 346e8d8bef9SDimitry Andric VPBranchOnMaskSC, 347bdd1243dSDimitry Andric VPDerivedIVSC, 34881ad6265SDimitry Andric VPExpandSCEVSC, 349e8d8bef9SDimitry Andric VPInstructionSC, 350e8d8bef9SDimitry Andric VPInterleaveSC, 351e8d8bef9SDimitry Andric VPReductionSC, 352e8d8bef9SDimitry Andric VPReplicateSC, 35381ad6265SDimitry Andric VPScalarIVStepsSC, 354647cbc5dSDimitry Andric VPVectorPointerSC, 355e8d8bef9SDimitry Andric VPWidenCallSC, 35604eeddc0SDimitry Andric VPWidenCanonicalIVSC, 35706c3fb27SDimitry Andric VPWidenCastSC, 358e8d8bef9SDimitry Andric VPWidenGEPSC, 359e8d8bef9SDimitry Andric VPWidenMemoryInstructionSC, 360e8d8bef9SDimitry Andric VPWidenSC, 361fe6060f1SDimitry Andric VPWidenSelectSC, 36206c3fb27SDimitry Andric // START: Phi-like recipes. Need to be kept together. 363fe6060f1SDimitry Andric VPBlendSC, 364bdd1243dSDimitry Andric VPPredInstPHISC, 36506c3fb27SDimitry Andric // START: SubclassID for recipes that inherit VPHeaderPHIRecipe. 36606c3fb27SDimitry Andric // VPHeaderPHIRecipe need to be kept together. 36704eeddc0SDimitry Andric VPCanonicalIVPHISC, 368753f127fSDimitry Andric VPActiveLaneMaskPHISC, 369fe6060f1SDimitry Andric VPFirstOrderRecurrencePHISC, 370fe6060f1SDimitry Andric VPWidenPHISC, 371fe6060f1SDimitry Andric VPWidenIntOrFpInductionSC, 37281ad6265SDimitry Andric VPWidenPointerInductionSC, 373fe6060f1SDimitry Andric VPReductionPHISC, 37406c3fb27SDimitry Andric // END: SubclassID for recipes that inherit VPHeaderPHIRecipe 37506c3fb27SDimitry Andric // END: Phi-like recipes 376fe6060f1SDimitry Andric VPFirstPHISC = VPBlendSC, 377bdd1243dSDimitry Andric VPFirstHeaderPHISC = VPCanonicalIVPHISC, 37806c3fb27SDimitry Andric VPLastHeaderPHISC = VPReductionPHISC, 379fe6060f1SDimitry Andric VPLastPHISC = VPReductionPHISC, 380e8d8bef9SDimitry Andric }; 381e8d8bef9SDimitry Andric VPDef(const unsigned char SC)382e8d8bef9SDimitry Andric VPDef(const unsigned char SC) : SubclassID(SC) {} 383e8d8bef9SDimitry Andric ~VPDef()384e8d8bef9SDimitry Andric virtual ~VPDef() { 385e8d8bef9SDimitry Andric for (VPValue *D : make_early_inc_range(DefinedValues)) { 386e8d8bef9SDimitry Andric assert(D->Def == this && 387e8d8bef9SDimitry Andric "all defined VPValues should point to the containing VPDef"); 388e8d8bef9SDimitry Andric assert(D->getNumUsers() == 0 && 389e8d8bef9SDimitry Andric "all defined VPValues should have no more users"); 390e8d8bef9SDimitry Andric D->Def = nullptr; 391e8d8bef9SDimitry Andric delete D; 392e8d8bef9SDimitry Andric } 393e8d8bef9SDimitry Andric } 394e8d8bef9SDimitry Andric 395fe6060f1SDimitry Andric /// Returns the only VPValue defined by the VPDef. Can only be called for 396fe6060f1SDimitry Andric /// VPDefs with a single defined value. getVPSingleValue()397fe6060f1SDimitry Andric VPValue *getVPSingleValue() { 398fe6060f1SDimitry Andric assert(DefinedValues.size() == 1 && "must have exactly one defined value"); 399fe6060f1SDimitry Andric assert(DefinedValues[0] && "defined value must be non-null"); 400fe6060f1SDimitry Andric return DefinedValues[0]; 401fe6060f1SDimitry Andric } getVPSingleValue()402fe6060f1SDimitry Andric const VPValue *getVPSingleValue() const { 403fe6060f1SDimitry Andric assert(DefinedValues.size() == 1 && "must have exactly one defined value"); 404fe6060f1SDimitry Andric assert(DefinedValues[0] && "defined value must be non-null"); 405fe6060f1SDimitry Andric return DefinedValues[0]; 406fe6060f1SDimitry Andric } 407fe6060f1SDimitry Andric 408e8d8bef9SDimitry Andric /// Returns the VPValue with index \p I defined by the VPDef. getVPValue(unsigned I)409fe6060f1SDimitry Andric VPValue *getVPValue(unsigned I) { 410e8d8bef9SDimitry Andric assert(DefinedValues[I] && "defined value must be non-null"); 411e8d8bef9SDimitry Andric return DefinedValues[I]; 412e8d8bef9SDimitry Andric } getVPValue(unsigned I)413fe6060f1SDimitry Andric const VPValue *getVPValue(unsigned I) const { 414e8d8bef9SDimitry Andric assert(DefinedValues[I] && "defined value must be non-null"); 415e8d8bef9SDimitry Andric return DefinedValues[I]; 416e8d8bef9SDimitry Andric } 417e8d8bef9SDimitry Andric 418e8d8bef9SDimitry Andric /// Returns an ArrayRef of the values defined by the VPDef. definedValues()419e8d8bef9SDimitry Andric ArrayRef<VPValue *> definedValues() { return DefinedValues; } 420e8d8bef9SDimitry Andric /// Returns an ArrayRef of the values defined by the VPDef. definedValues()421e8d8bef9SDimitry Andric ArrayRef<VPValue *> definedValues() const { return DefinedValues; } 422e8d8bef9SDimitry Andric 423e8d8bef9SDimitry Andric /// Returns the number of values defined by the VPDef. getNumDefinedValues()424e8d8bef9SDimitry Andric unsigned getNumDefinedValues() const { return DefinedValues.size(); } 425e8d8bef9SDimitry Andric 426e8d8bef9SDimitry Andric /// \return an ID for the concrete type of this object. 427e8d8bef9SDimitry Andric /// This is used to implement the classof checks. This should not be used 428e8d8bef9SDimitry Andric /// for any other purpose, as the values may change as LLVM evolves. getVPDefID()429e8d8bef9SDimitry Andric unsigned getVPDefID() const { return SubclassID; } 430e8d8bef9SDimitry Andric 431fe6060f1SDimitry Andric #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 432e8d8bef9SDimitry Andric /// Dump the VPDef to stderr (for debugging). 433e8d8bef9SDimitry Andric void dump() const; 434e8d8bef9SDimitry Andric 435e8d8bef9SDimitry Andric /// Each concrete VPDef prints itself. 436e8d8bef9SDimitry Andric virtual void print(raw_ostream &O, const Twine &Indent, 437e8d8bef9SDimitry Andric VPSlotTracker &SlotTracker) const = 0; 438fe6060f1SDimitry Andric #endif 439e8d8bef9SDimitry Andric }; 440e8d8bef9SDimitry Andric 4415ffd83dbSDimitry Andric class VPlan; 4425ffd83dbSDimitry Andric class VPBasicBlock; 4435ffd83dbSDimitry Andric 4445ffd83dbSDimitry Andric /// This class can be used to assign consecutive numbers to all VPValues in a 4455ffd83dbSDimitry Andric /// VPlan and allows querying the numbering for printing, similar to the 4465ffd83dbSDimitry Andric /// ModuleSlotTracker for IR values. 4475ffd83dbSDimitry Andric class VPSlotTracker { 4485ffd83dbSDimitry Andric DenseMap<const VPValue *, unsigned> Slots; 4495ffd83dbSDimitry Andric unsigned NextSlot = 0; 4505ffd83dbSDimitry Andric 4515ffd83dbSDimitry Andric void assignSlot(const VPValue *V); 4525ffd83dbSDimitry Andric void assignSlots(const VPlan &Plan); 45306c3fb27SDimitry Andric void assignSlots(const VPBasicBlock *VPBB); 4545ffd83dbSDimitry Andric 4555ffd83dbSDimitry Andric public: 456e8d8bef9SDimitry Andric VPSlotTracker(const VPlan *Plan = nullptr) { 4575ffd83dbSDimitry Andric if (Plan) 4585ffd83dbSDimitry Andric assignSlots(*Plan); 4595ffd83dbSDimitry Andric } 4605ffd83dbSDimitry Andric getSlot(const VPValue * V)4615ffd83dbSDimitry Andric unsigned getSlot(const VPValue *V) const { 4625ffd83dbSDimitry Andric auto I = Slots.find(V); 4635ffd83dbSDimitry Andric if (I == Slots.end()) 4645ffd83dbSDimitry Andric return -1; 4655ffd83dbSDimitry Andric return I->second; 4665ffd83dbSDimitry Andric } 4675ffd83dbSDimitry Andric }; 4680b57cec5SDimitry Andric 4690b57cec5SDimitry Andric } // namespace llvm 4700b57cec5SDimitry Andric 4710b57cec5SDimitry Andric #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H 472