1 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// Interface for Targets to specify which operations they can successfully 10 /// select and how the others should be expanded most efficiently. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 15 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 16 17 #include "llvm/ADT/DenseMap.h" 18 #include "llvm/ADT/STLExtras.h" 19 #include "llvm/ADT/SmallBitVector.h" 20 #include "llvm/ADT/SmallVector.h" 21 #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" 22 #include "llvm/CodeGen/MachineFunction.h" 23 #include "llvm/CodeGen/TargetOpcodes.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/LowLevelTypeImpl.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include <cassert> 28 #include <cstdint> 29 #include <tuple> 30 #include <unordered_map> 31 #include <utility> 32 33 namespace llvm { 34 35 extern cl::opt<bool> DisableGISelLegalityCheck; 36 37 class LegalizerHelper; 38 class MachineInstr; 39 class MachineRegisterInfo; 40 class MCInstrInfo; 41 42 namespace LegalizeActions { 43 enum LegalizeAction : std::uint8_t { 44 /// The operation is expected to be selectable directly by the target, and 45 /// no transformation is necessary. 46 Legal, 47 48 /// The operation should be synthesized from multiple instructions acting on 49 /// a narrower scalar base-type. For example a 64-bit add might be 50 /// implemented in terms of 32-bit add-with-carry. 51 NarrowScalar, 52 53 /// The operation should be implemented in terms of a wider scalar 54 /// base-type. For example a <2 x s8> add could be implemented as a <2 55 /// x s32> add (ignoring the high bits). 56 WidenScalar, 57 58 /// The (vector) operation should be implemented by splitting it into 59 /// sub-vectors where the operation is legal. For example a <8 x s64> add 60 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover 61 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add 62 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover 63 /// types can be avoided by doing MoreElements first. 64 FewerElements, 65 66 /// The (vector) operation should be implemented by widening the input 67 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is 68 /// rarely legal, but you might perform an <8 x i8> and then only look at 69 /// the first two results. 70 MoreElements, 71 72 /// Perform the operation on a different, but equivalently sized type. 73 Bitcast, 74 75 /// The operation itself must be expressed in terms of simpler actions on 76 /// this target. E.g. a SREM replaced by an SDIV and subtraction. 77 Lower, 78 79 /// The operation should be implemented as a call to some kind of runtime 80 /// support library. For example this usually happens on machines that don't 81 /// support floating-point operations natively. 82 Libcall, 83 84 /// The target wants to do something special with this combination of 85 /// operand and type. A callback will be issued when it is needed. 86 Custom, 87 88 /// This operation is completely unsupported on the target. A programming 89 /// error has occurred. 90 Unsupported, 91 92 /// Sentinel value for when no action was found in the specified table. 93 NotFound, 94 95 /// Fall back onto the old rules. 96 /// TODO: Remove this once we've migrated 97 UseLegacyRules, 98 }; 99 } // end namespace LegalizeActions 100 raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); 101 102 using LegalizeActions::LegalizeAction; 103 104 /// The LegalityQuery object bundles together all the information that's needed 105 /// to decide whether a given operation is legal or not. 106 /// For efficiency, it doesn't make a copy of Types so care must be taken not 107 /// to free it before using the query. 108 struct LegalityQuery { 109 unsigned Opcode; 110 ArrayRef<LLT> Types; 111 112 struct MemDesc { 113 LLT MemoryTy; 114 uint64_t AlignInBits; 115 AtomicOrdering Ordering; 116 117 MemDesc() = default; 118 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering) 119 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {} 120 MemDesc(const MachineMemOperand &MMO) 121 : MemoryTy(MMO.getMemoryType()), 122 AlignInBits(MMO.getAlign().value() * 8), 123 Ordering(MMO.getSuccessOrdering()) {} 124 }; 125 126 /// Operations which require memory can use this to place requirements on the 127 /// memory type for each MMO. 128 ArrayRef<MemDesc> MMODescrs; 129 130 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, 131 const ArrayRef<MemDesc> MMODescrs) 132 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} 133 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) 134 : LegalityQuery(Opcode, Types, {}) {} 135 136 raw_ostream &print(raw_ostream &OS) const; 137 }; 138 139 /// The result of a query. It either indicates a final answer of Legal or 140 /// Unsupported or describes an action that must be taken to make an operation 141 /// more legal. 142 struct LegalizeActionStep { 143 /// The action to take or the final answer. 144 LegalizeAction Action; 145 /// If describing an action, the type index to change. Otherwise zero. 146 unsigned TypeIdx; 147 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. 148 LLT NewType; 149 150 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, 151 const LLT NewType) 152 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} 153 154 LegalizeActionStep(LegacyLegalizeActionStep Step) 155 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { 156 switch (Step.Action) { 157 case LegacyLegalizeActions::Legal: 158 Action = LegalizeActions::Legal; 159 break; 160 case LegacyLegalizeActions::NarrowScalar: 161 Action = LegalizeActions::NarrowScalar; 162 break; 163 case LegacyLegalizeActions::WidenScalar: 164 Action = LegalizeActions::WidenScalar; 165 break; 166 case LegacyLegalizeActions::FewerElements: 167 Action = LegalizeActions::FewerElements; 168 break; 169 case LegacyLegalizeActions::MoreElements: 170 Action = LegalizeActions::MoreElements; 171 break; 172 case LegacyLegalizeActions::Bitcast: 173 Action = LegalizeActions::Bitcast; 174 break; 175 case LegacyLegalizeActions::Lower: 176 Action = LegalizeActions::Lower; 177 break; 178 case LegacyLegalizeActions::Libcall: 179 Action = LegalizeActions::Libcall; 180 break; 181 case LegacyLegalizeActions::Custom: 182 Action = LegalizeActions::Custom; 183 break; 184 case LegacyLegalizeActions::Unsupported: 185 Action = LegalizeActions::Unsupported; 186 break; 187 case LegacyLegalizeActions::NotFound: 188 Action = LegalizeActions::NotFound; 189 break; 190 } 191 } 192 193 bool operator==(const LegalizeActionStep &RHS) const { 194 return std::tie(Action, TypeIdx, NewType) == 195 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); 196 } 197 }; 198 199 using LegalityPredicate = std::function<bool (const LegalityQuery &)>; 200 using LegalizeMutation = 201 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; 202 203 namespace LegalityPredicates { 204 struct TypePairAndMemDesc { 205 LLT Type0; 206 LLT Type1; 207 LLT MemTy; 208 uint64_t Align; 209 210 bool operator==(const TypePairAndMemDesc &Other) const { 211 return Type0 == Other.Type0 && Type1 == Other.Type1 && 212 Align == Other.Align && MemTy == Other.MemTy; 213 } 214 215 /// \returns true if this memory access is legal with for the access described 216 /// by \p Other (The alignment is sufficient for the size and result type). 217 bool isCompatible(const TypePairAndMemDesc &Other) const { 218 return Type0 == Other.Type0 && Type1 == Other.Type1 && 219 Align >= Other.Align && 220 // FIXME: This perhaps should be stricter, but the current legality 221 // rules are written only considering the size. 222 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); 223 } 224 }; 225 226 /// True iff P0 and P1 are true. 227 template<typename Predicate> 228 Predicate all(Predicate P0, Predicate P1) { 229 return [=](const LegalityQuery &Query) { 230 return P0(Query) && P1(Query); 231 }; 232 } 233 /// True iff all given predicates are true. 234 template<typename Predicate, typename... Args> 235 Predicate all(Predicate P0, Predicate P1, Args... args) { 236 return all(all(P0, P1), args...); 237 } 238 239 /// True iff P0 or P1 are true. 240 template<typename Predicate> 241 Predicate any(Predicate P0, Predicate P1) { 242 return [=](const LegalityQuery &Query) { 243 return P0(Query) || P1(Query); 244 }; 245 } 246 /// True iff any given predicates are true. 247 template<typename Predicate, typename... Args> 248 Predicate any(Predicate P0, Predicate P1, Args... args) { 249 return any(any(P0, P1), args...); 250 } 251 252 /// True iff the given type index is the specified type. 253 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); 254 /// True iff the given type index is one of the specified types. 255 LegalityPredicate typeInSet(unsigned TypeIdx, 256 std::initializer_list<LLT> TypesInit); 257 258 /// True iff the given type index is not the specified type. 259 inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { 260 return [=](const LegalityQuery &Query) { 261 return Query.Types[TypeIdx] != Type; 262 }; 263 } 264 265 /// True iff the given types for the given pair of type indexes is one of the 266 /// specified type pairs. 267 LegalityPredicate 268 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, 269 std::initializer_list<std::pair<LLT, LLT>> TypesInit); 270 /// True iff the given types for the given pair of type indexes is one of the 271 /// specified type pairs. 272 LegalityPredicate typePairAndMemDescInSet( 273 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, 274 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); 275 /// True iff the specified type index is a scalar. 276 LegalityPredicate isScalar(unsigned TypeIdx); 277 /// True iff the specified type index is a vector. 278 LegalityPredicate isVector(unsigned TypeIdx); 279 /// True iff the specified type index is a pointer (with any address space). 280 LegalityPredicate isPointer(unsigned TypeIdx); 281 /// True iff the specified type index is a pointer with the specified address 282 /// space. 283 LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); 284 285 /// True if the type index is a vector with element type \p EltTy 286 LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy); 287 288 /// True iff the specified type index is a scalar that's narrower than the given 289 /// size. 290 LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size); 291 292 /// True iff the specified type index is a scalar that's wider than the given 293 /// size. 294 LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size); 295 296 /// True iff the specified type index is a scalar or vector with an element type 297 /// that's narrower than the given size. 298 LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); 299 300 /// True iff the specified type index is a scalar or a vector with an element 301 /// type that's wider than the given size. 302 LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); 303 304 /// True iff the specified type index is a scalar whose size is not a multiple 305 /// of Size. 306 LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size); 307 308 /// True iff the specified type index is a scalar whose size is not a power of 309 /// 2. 310 LegalityPredicate sizeNotPow2(unsigned TypeIdx); 311 312 /// True iff the specified type index is a scalar or vector whose element size 313 /// is not a power of 2. 314 LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); 315 316 /// True if the total bitwidth of the specified type index is \p Size bits. 317 LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size); 318 319 /// True iff the specified type indices are both the same bit size. 320 LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); 321 322 /// True iff the first type index has a larger total bit size than second type 323 /// index. 324 LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); 325 326 /// True iff the first type index has a smaller total bit size than second type 327 /// index. 328 LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); 329 330 /// True iff the specified MMO index has a size that is not a power of 2 331 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); 332 /// True iff the specified type index is a vector whose element count is not a 333 /// power of 2. 334 LegalityPredicate numElementsNotPow2(unsigned TypeIdx); 335 /// True iff the specified MMO index has at an atomic ordering of at Ordering or 336 /// stronger. 337 LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, 338 AtomicOrdering Ordering); 339 } // end namespace LegalityPredicates 340 341 namespace LegalizeMutations { 342 /// Select this specific type for the given type index. 343 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); 344 345 /// Keep the same type as the given type index. 346 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); 347 348 /// Keep the same scalar or element type as the given type index. 349 LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); 350 351 /// Keep the same scalar or element type as the given type. 352 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); 353 354 /// Change the scalar size or element size to have the same scalar size as type 355 /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and 356 /// only changes the size. 357 LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx); 358 359 /// Widen the scalar type or vector element type for the given type index to the 360 /// next power of 2. 361 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); 362 363 /// Widen the scalar type or vector element type for the given type index to 364 /// next multiple of \p Size. 365 LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx, 366 unsigned Size); 367 368 /// Add more elements to the type for the given type index to the next power of 369 /// 2. 370 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); 371 /// Break up the vector type for the given type index into the element type. 372 LegalizeMutation scalarize(unsigned TypeIdx); 373 } // end namespace LegalizeMutations 374 375 /// A single rule in a legalizer info ruleset. 376 /// The specified action is chosen when the predicate is true. Where appropriate 377 /// for the action (e.g. for WidenScalar) the new type is selected using the 378 /// given mutator. 379 class LegalizeRule { 380 LegalityPredicate Predicate; 381 LegalizeAction Action; 382 LegalizeMutation Mutation; 383 384 public: 385 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, 386 LegalizeMutation Mutation = nullptr) 387 : Predicate(Predicate), Action(Action), Mutation(Mutation) {} 388 389 /// Test whether the LegalityQuery matches. 390 bool match(const LegalityQuery &Query) const { 391 return Predicate(Query); 392 } 393 394 LegalizeAction getAction() const { return Action; } 395 396 /// Determine the change to make. 397 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { 398 if (Mutation) 399 return Mutation(Query); 400 return std::make_pair(0, LLT{}); 401 } 402 }; 403 404 class LegalizeRuleSet { 405 /// When non-zero, the opcode we are an alias of 406 unsigned AliasOf = 0; 407 /// If true, there is another opcode that aliases this one 408 bool IsAliasedByAnother = false; 409 SmallVector<LegalizeRule, 2> Rules; 410 411 #ifndef NDEBUG 412 /// If bit I is set, this rule set contains a rule that may handle (predicate 413 /// or perform an action upon (or both)) the type index I. The uncertainty 414 /// comes from free-form rules executing user-provided lambda functions. We 415 /// conservatively assume such rules do the right thing and cover all type 416 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs 417 /// to be to distinguish such cases from the cases where all type indices are 418 /// individually handled. 419 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - 420 MCOI::OPERAND_FIRST_GENERIC + 2}; 421 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - 422 MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; 423 #endif 424 425 unsigned typeIdx(unsigned TypeIdx) { 426 assert(TypeIdx <= 427 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && 428 "Type Index is out of bounds"); 429 #ifndef NDEBUG 430 TypeIdxsCovered.set(TypeIdx); 431 #endif 432 return TypeIdx; 433 } 434 435 void markAllIdxsAsCovered() { 436 #ifndef NDEBUG 437 TypeIdxsCovered.set(); 438 ImmIdxsCovered.set(); 439 #endif 440 } 441 442 void add(const LegalizeRule &Rule) { 443 assert(AliasOf == 0 && 444 "RuleSet is aliased, change the representative opcode instead"); 445 Rules.push_back(Rule); 446 } 447 448 static bool always(const LegalityQuery &) { return true; } 449 450 /// Use the given action when the predicate is true. 451 /// Action should not be an action that requires mutation. 452 LegalizeRuleSet &actionIf(LegalizeAction Action, 453 LegalityPredicate Predicate) { 454 add({Predicate, Action}); 455 return *this; 456 } 457 /// Use the given action when the predicate is true. 458 /// Action should be an action that requires mutation. 459 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, 460 LegalizeMutation Mutation) { 461 add({Predicate, Action, Mutation}); 462 return *this; 463 } 464 /// Use the given action when type index 0 is any type in the given list. 465 /// Action should not be an action that requires mutation. 466 LegalizeRuleSet &actionFor(LegalizeAction Action, 467 std::initializer_list<LLT> Types) { 468 using namespace LegalityPredicates; 469 return actionIf(Action, typeInSet(typeIdx(0), Types)); 470 } 471 /// Use the given action when type index 0 is any type in the given list. 472 /// Action should be an action that requires mutation. 473 LegalizeRuleSet &actionFor(LegalizeAction Action, 474 std::initializer_list<LLT> Types, 475 LegalizeMutation Mutation) { 476 using namespace LegalityPredicates; 477 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation); 478 } 479 /// Use the given action when type indexes 0 and 1 is any type pair in the 480 /// given list. 481 /// Action should not be an action that requires mutation. 482 LegalizeRuleSet &actionFor(LegalizeAction Action, 483 std::initializer_list<std::pair<LLT, LLT>> Types) { 484 using namespace LegalityPredicates; 485 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 486 } 487 /// Use the given action when type indexes 0 and 1 is any type pair in the 488 /// given list. 489 /// Action should be an action that requires mutation. 490 LegalizeRuleSet &actionFor(LegalizeAction Action, 491 std::initializer_list<std::pair<LLT, LLT>> Types, 492 LegalizeMutation Mutation) { 493 using namespace LegalityPredicates; 494 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), 495 Mutation); 496 } 497 /// Use the given action when type index 0 is any type in the given list and 498 /// imm index 0 is anything. Action should not be an action that requires 499 /// mutation. 500 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, 501 std::initializer_list<LLT> Types) { 502 using namespace LegalityPredicates; 503 immIdx(0); // Inform verifier imm idx 0 is handled. 504 return actionIf(Action, typeInSet(typeIdx(0), Types)); 505 } 506 507 LegalizeRuleSet &actionForTypeWithAnyImm( 508 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) { 509 using namespace LegalityPredicates; 510 immIdx(0); // Inform verifier imm idx 0 is handled. 511 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 512 } 513 514 /// Use the given action when type indexes 0 and 1 are both in the given list. 515 /// That is, the type pair is in the cartesian product of the list. 516 /// Action should not be an action that requires mutation. 517 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, 518 std::initializer_list<LLT> Types) { 519 using namespace LegalityPredicates; 520 return actionIf(Action, all(typeInSet(typeIdx(0), Types), 521 typeInSet(typeIdx(1), Types))); 522 } 523 /// Use the given action when type indexes 0 and 1 are both in their 524 /// respective lists. 525 /// That is, the type pair is in the cartesian product of the lists 526 /// Action should not be an action that requires mutation. 527 LegalizeRuleSet & 528 actionForCartesianProduct(LegalizeAction Action, 529 std::initializer_list<LLT> Types0, 530 std::initializer_list<LLT> Types1) { 531 using namespace LegalityPredicates; 532 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 533 typeInSet(typeIdx(1), Types1))); 534 } 535 /// Use the given action when type indexes 0, 1, and 2 are all in their 536 /// respective lists. 537 /// That is, the type triple is in the cartesian product of the lists 538 /// Action should not be an action that requires mutation. 539 LegalizeRuleSet &actionForCartesianProduct( 540 LegalizeAction Action, std::initializer_list<LLT> Types0, 541 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { 542 using namespace LegalityPredicates; 543 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 544 all(typeInSet(typeIdx(1), Types1), 545 typeInSet(typeIdx(2), Types2)))); 546 } 547 548 public: 549 LegalizeRuleSet() = default; 550 551 bool isAliasedByAnother() { return IsAliasedByAnother; } 552 void setIsAliasedByAnother() { IsAliasedByAnother = true; } 553 void aliasTo(unsigned Opcode) { 554 assert((AliasOf == 0 || AliasOf == Opcode) && 555 "Opcode is already aliased to another opcode"); 556 assert(Rules.empty() && "Aliasing will discard rules"); 557 AliasOf = Opcode; 558 } 559 unsigned getAlias() const { return AliasOf; } 560 561 unsigned immIdx(unsigned ImmIdx) { 562 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - 563 MCOI::OPERAND_FIRST_GENERIC_IMM) && 564 "Imm Index is out of bounds"); 565 #ifndef NDEBUG 566 ImmIdxsCovered.set(ImmIdx); 567 #endif 568 return ImmIdx; 569 } 570 571 /// The instruction is legal if predicate is true. 572 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { 573 // We have no choice but conservatively assume that the free-form 574 // user-provided Predicate properly handles all type indices: 575 markAllIdxsAsCovered(); 576 return actionIf(LegalizeAction::Legal, Predicate); 577 } 578 /// The instruction is legal when type index 0 is any type in the given list. 579 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { 580 return actionFor(LegalizeAction::Legal, Types); 581 } 582 /// The instruction is legal when type indexes 0 and 1 is any type pair in the 583 /// given list. 584 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 585 return actionFor(LegalizeAction::Legal, Types); 586 } 587 /// The instruction is legal when type index 0 is any type in the given list 588 /// and imm index 0 is anything. 589 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { 590 markAllIdxsAsCovered(); 591 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 592 } 593 594 LegalizeRuleSet &legalForTypeWithAnyImm( 595 std::initializer_list<std::pair<LLT, LLT>> Types) { 596 markAllIdxsAsCovered(); 597 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 598 } 599 600 /// The instruction is legal when type indexes 0 and 1 along with the memory 601 /// size and minimum alignment is any type and size tuple in the given list. 602 LegalizeRuleSet &legalForTypesWithMemDesc( 603 std::initializer_list<LegalityPredicates::TypePairAndMemDesc> 604 TypesAndMemDesc) { 605 return actionIf(LegalizeAction::Legal, 606 LegalityPredicates::typePairAndMemDescInSet( 607 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); 608 } 609 /// The instruction is legal when type indexes 0 and 1 are both in the given 610 /// list. That is, the type pair is in the cartesian product of the list. 611 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { 612 return actionForCartesianProduct(LegalizeAction::Legal, Types); 613 } 614 /// The instruction is legal when type indexes 0 and 1 are both their 615 /// respective lists. 616 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 617 std::initializer_list<LLT> Types1) { 618 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); 619 } 620 /// The instruction is legal when type indexes 0, 1, and 2 are both their 621 /// respective lists. 622 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 623 std::initializer_list<LLT> Types1, 624 std::initializer_list<LLT> Types2) { 625 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, 626 Types2); 627 } 628 629 LegalizeRuleSet &alwaysLegal() { 630 using namespace LegalizeMutations; 631 markAllIdxsAsCovered(); 632 return actionIf(LegalizeAction::Legal, always); 633 } 634 635 /// The specified type index is coerced if predicate is true. 636 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, 637 LegalizeMutation Mutation) { 638 // We have no choice but conservatively assume that lowering with a 639 // free-form user provided Predicate properly handles all type indices: 640 markAllIdxsAsCovered(); 641 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation); 642 } 643 644 /// The instruction is lowered. 645 LegalizeRuleSet &lower() { 646 using namespace LegalizeMutations; 647 // We have no choice but conservatively assume that predicate-less lowering 648 // properly handles all type indices by design: 649 markAllIdxsAsCovered(); 650 return actionIf(LegalizeAction::Lower, always); 651 } 652 /// The instruction is lowered if predicate is true. Keep type index 0 as the 653 /// same type. 654 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { 655 using namespace LegalizeMutations; 656 // We have no choice but conservatively assume that lowering with a 657 // free-form user provided Predicate properly handles all type indices: 658 markAllIdxsAsCovered(); 659 return actionIf(LegalizeAction::Lower, Predicate); 660 } 661 /// The instruction is lowered if predicate is true. 662 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, 663 LegalizeMutation Mutation) { 664 // We have no choice but conservatively assume that lowering with a 665 // free-form user provided Predicate properly handles all type indices: 666 markAllIdxsAsCovered(); 667 return actionIf(LegalizeAction::Lower, Predicate, Mutation); 668 } 669 /// The instruction is lowered when type index 0 is any type in the given 670 /// list. Keep type index 0 as the same type. 671 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { 672 return actionFor(LegalizeAction::Lower, Types); 673 } 674 /// The instruction is lowered when type index 0 is any type in the given 675 /// list. 676 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, 677 LegalizeMutation Mutation) { 678 return actionFor(LegalizeAction::Lower, Types, Mutation); 679 } 680 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 681 /// the given list. Keep type index 0 as the same type. 682 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 683 return actionFor(LegalizeAction::Lower, Types); 684 } 685 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 686 /// the given list. 687 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, 688 LegalizeMutation Mutation) { 689 return actionFor(LegalizeAction::Lower, Types, Mutation); 690 } 691 /// The instruction is lowered when type indexes 0 and 1 are both in their 692 /// respective lists. 693 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 694 std::initializer_list<LLT> Types1) { 695 using namespace LegalityPredicates; 696 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1); 697 } 698 /// The instruction is lowered when when type indexes 0, 1, and 2 are all in 699 /// their respective lists. 700 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 701 std::initializer_list<LLT> Types1, 702 std::initializer_list<LLT> Types2) { 703 using namespace LegalityPredicates; 704 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1, 705 Types2); 706 } 707 708 /// The instruction is emitted as a library call. 709 LegalizeRuleSet &libcall() { 710 using namespace LegalizeMutations; 711 // We have no choice but conservatively assume that predicate-less lowering 712 // properly handles all type indices by design: 713 markAllIdxsAsCovered(); 714 return actionIf(LegalizeAction::Libcall, always); 715 } 716 717 /// Like legalIf, but for the Libcall action. 718 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { 719 // We have no choice but conservatively assume that a libcall with a 720 // free-form user provided Predicate properly handles all type indices: 721 markAllIdxsAsCovered(); 722 return actionIf(LegalizeAction::Libcall, Predicate); 723 } 724 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { 725 return actionFor(LegalizeAction::Libcall, Types); 726 } 727 LegalizeRuleSet & 728 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 729 return actionFor(LegalizeAction::Libcall, Types); 730 } 731 LegalizeRuleSet & 732 libcallForCartesianProduct(std::initializer_list<LLT> Types) { 733 return actionForCartesianProduct(LegalizeAction::Libcall, Types); 734 } 735 LegalizeRuleSet & 736 libcallForCartesianProduct(std::initializer_list<LLT> Types0, 737 std::initializer_list<LLT> Types1) { 738 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); 739 } 740 741 /// Widen the scalar to the one selected by the mutation if the predicate is 742 /// true. 743 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, 744 LegalizeMutation Mutation) { 745 // We have no choice but conservatively assume that an action with a 746 // free-form user provided Predicate properly handles all type indices: 747 markAllIdxsAsCovered(); 748 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); 749 } 750 /// Narrow the scalar to the one selected by the mutation if the predicate is 751 /// true. 752 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, 753 LegalizeMutation Mutation) { 754 // We have no choice but conservatively assume that an action with a 755 // free-form user provided Predicate properly handles all type indices: 756 markAllIdxsAsCovered(); 757 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); 758 } 759 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any 760 /// type pair in the given list. 761 LegalizeRuleSet & 762 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types, 763 LegalizeMutation Mutation) { 764 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation); 765 } 766 767 /// Add more elements to reach the type selected by the mutation if the 768 /// predicate is true. 769 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, 770 LegalizeMutation Mutation) { 771 // We have no choice but conservatively assume that an action with a 772 // free-form user provided Predicate properly handles all type indices: 773 markAllIdxsAsCovered(); 774 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); 775 } 776 /// Remove elements to reach the type selected by the mutation if the 777 /// predicate is true. 778 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, 779 LegalizeMutation Mutation) { 780 // We have no choice but conservatively assume that an action with a 781 // free-form user provided Predicate properly handles all type indices: 782 markAllIdxsAsCovered(); 783 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); 784 } 785 786 /// The instruction is unsupported. 787 LegalizeRuleSet &unsupported() { 788 markAllIdxsAsCovered(); 789 return actionIf(LegalizeAction::Unsupported, always); 790 } 791 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { 792 return actionIf(LegalizeAction::Unsupported, Predicate); 793 } 794 795 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) { 796 return actionFor(LegalizeAction::Unsupported, Types); 797 } 798 799 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { 800 return actionIf(LegalizeAction::Unsupported, 801 LegalityPredicates::memSizeInBytesNotPow2(0)); 802 } 803 LegalizeRuleSet &lowerIfMemSizeNotPow2() { 804 return actionIf(LegalizeAction::Lower, 805 LegalityPredicates::memSizeInBytesNotPow2(0)); 806 } 807 808 LegalizeRuleSet &customIf(LegalityPredicate Predicate) { 809 // We have no choice but conservatively assume that a custom action with a 810 // free-form user provided Predicate properly handles all type indices: 811 markAllIdxsAsCovered(); 812 return actionIf(LegalizeAction::Custom, Predicate); 813 } 814 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { 815 return actionFor(LegalizeAction::Custom, Types); 816 } 817 818 /// The instruction is custom when type indexes 0 and 1 is any type pair in the 819 /// given list. 820 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 821 return actionFor(LegalizeAction::Custom, Types); 822 } 823 824 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { 825 return actionForCartesianProduct(LegalizeAction::Custom, Types); 826 } 827 /// The instruction is custom when type indexes 0 and 1 are both in their 828 /// respective lists. 829 LegalizeRuleSet & 830 customForCartesianProduct(std::initializer_list<LLT> Types0, 831 std::initializer_list<LLT> Types1) { 832 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); 833 } 834 /// The instruction is custom when when type indexes 0, 1, and 2 are all in 835 /// their respective lists. 836 LegalizeRuleSet & 837 customForCartesianProduct(std::initializer_list<LLT> Types0, 838 std::initializer_list<LLT> Types1, 839 std::initializer_list<LLT> Types2) { 840 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1, 841 Types2); 842 } 843 844 /// Unconditionally custom lower. 845 LegalizeRuleSet &custom() { 846 return customIf(always); 847 } 848 849 /// Widen the scalar to the next power of two that is at least MinSize. 850 /// No effect if the type is not a scalar or is a power of two. 851 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, 852 unsigned MinSize = 0) { 853 using namespace LegalityPredicates; 854 return actionIf( 855 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), 856 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 857 } 858 859 /// Widen the scalar to the next multiple of Size. No effect if the 860 /// type is not a scalar or is a multiple of Size. 861 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx, 862 unsigned Size) { 863 using namespace LegalityPredicates; 864 return actionIf( 865 LegalizeAction::WidenScalar, sizeNotMultipleOf(typeIdx(TypeIdx), Size), 866 LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size)); 867 } 868 869 /// Widen the scalar or vector element type to the next power of two that is 870 /// at least MinSize. No effect if the scalar size is a power of two. 871 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, 872 unsigned MinSize = 0) { 873 using namespace LegalityPredicates; 874 return actionIf( 875 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), 876 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 877 } 878 879 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { 880 using namespace LegalityPredicates; 881 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)), 882 Mutation); 883 } 884 885 LegalizeRuleSet &scalarize(unsigned TypeIdx) { 886 using namespace LegalityPredicates; 887 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), 888 LegalizeMutations::scalarize(TypeIdx)); 889 } 890 891 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) { 892 using namespace LegalityPredicates; 893 return actionIf(LegalizeAction::FewerElements, 894 all(Predicate, isVector(typeIdx(TypeIdx))), 895 LegalizeMutations::scalarize(TypeIdx)); 896 } 897 898 /// Ensure the scalar or element is at least as wide as Ty. 899 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) { 900 using namespace LegalityPredicates; 901 using namespace LegalizeMutations; 902 return actionIf(LegalizeAction::WidenScalar, 903 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), 904 changeElementTo(typeIdx(TypeIdx), Ty)); 905 } 906 907 /// Ensure the scalar or element is at least as wide as Ty. 908 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, 909 unsigned TypeIdx, const LLT Ty) { 910 using namespace LegalityPredicates; 911 using namespace LegalizeMutations; 912 return actionIf(LegalizeAction::WidenScalar, 913 all(Predicate, scalarOrEltNarrowerThan( 914 TypeIdx, Ty.getScalarSizeInBits())), 915 changeElementTo(typeIdx(TypeIdx), Ty)); 916 } 917 918 /// Ensure the scalar is at least as wide as Ty. 919 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) { 920 using namespace LegalityPredicates; 921 using namespace LegalizeMutations; 922 return actionIf(LegalizeAction::WidenScalar, 923 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()), 924 changeTo(typeIdx(TypeIdx), Ty)); 925 } 926 927 /// Ensure the scalar is at most as wide as Ty. 928 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) { 929 using namespace LegalityPredicates; 930 using namespace LegalizeMutations; 931 return actionIf(LegalizeAction::NarrowScalar, 932 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), 933 changeElementTo(typeIdx(TypeIdx), Ty)); 934 } 935 936 /// Ensure the scalar is at most as wide as Ty. 937 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) { 938 using namespace LegalityPredicates; 939 using namespace LegalizeMutations; 940 return actionIf(LegalizeAction::NarrowScalar, 941 scalarWiderThan(TypeIdx, Ty.getSizeInBits()), 942 changeTo(typeIdx(TypeIdx), Ty)); 943 } 944 945 /// Conditionally limit the maximum size of the scalar. 946 /// For example, when the maximum size of one type depends on the size of 947 /// another such as extracting N bits from an M bit container. 948 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 949 const LLT Ty) { 950 using namespace LegalityPredicates; 951 using namespace LegalizeMutations; 952 return actionIf( 953 LegalizeAction::NarrowScalar, 954 [=](const LegalityQuery &Query) { 955 const LLT QueryTy = Query.Types[TypeIdx]; 956 return QueryTy.isScalar() && 957 QueryTy.getSizeInBits() > Ty.getSizeInBits() && 958 Predicate(Query); 959 }, 960 changeElementTo(typeIdx(TypeIdx), Ty)); 961 } 962 963 /// Limit the range of scalar sizes to MinTy and MaxTy. 964 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy, 965 const LLT MaxTy) { 966 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); 967 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); 968 } 969 970 /// Limit the range of scalar sizes to MinTy and MaxTy. 971 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, 972 const LLT MaxTy) { 973 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); 974 } 975 976 /// Widen the scalar to match the size of another. 977 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { 978 typeIdx(TypeIdx); 979 return widenScalarIf( 980 [=](const LegalityQuery &Query) { 981 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 982 Query.Types[TypeIdx].getSizeInBits(); 983 }, 984 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx)); 985 } 986 987 /// Narrow the scalar to match the size of another. 988 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) { 989 typeIdx(TypeIdx); 990 return narrowScalarIf( 991 [=](const LegalityQuery &Query) { 992 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() < 993 Query.Types[TypeIdx].getSizeInBits(); 994 }, 995 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx)); 996 } 997 998 /// Change the type \p TypeIdx to have the same scalar size as type \p 999 /// SameSizeIdx. 1000 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) { 1001 return minScalarSameAs(TypeIdx, SameSizeIdx) 1002 .maxScalarSameAs(TypeIdx, SameSizeIdx); 1003 } 1004 1005 /// Conditionally widen the scalar or elt to match the size of another. 1006 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, 1007 unsigned TypeIdx, unsigned LargeTypeIdx) { 1008 typeIdx(TypeIdx); 1009 return widenScalarIf( 1010 [=](const LegalityQuery &Query) { 1011 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1012 Query.Types[TypeIdx].getScalarSizeInBits() && 1013 Predicate(Query); 1014 }, 1015 [=](const LegalityQuery &Query) { 1016 LLT T = Query.Types[LargeTypeIdx]; 1017 return std::make_pair(TypeIdx, T); 1018 }); 1019 } 1020 1021 /// Conditionally narrow the scalar or elt to match the size of another. 1022 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, 1023 unsigned TypeIdx, 1024 unsigned SmallTypeIdx) { 1025 typeIdx(TypeIdx); 1026 return narrowScalarIf( 1027 [=](const LegalityQuery &Query) { 1028 return Query.Types[SmallTypeIdx].getScalarSizeInBits() < 1029 Query.Types[TypeIdx].getScalarSizeInBits() && 1030 Predicate(Query); 1031 }, 1032 [=](const LegalityQuery &Query) { 1033 LLT T = Query.Types[SmallTypeIdx]; 1034 return std::make_pair(TypeIdx, T); 1035 }); 1036 } 1037 1038 /// Add more elements to the vector to reach the next power of two. 1039 /// No effect if the type is not a vector or the element count is a power of 1040 /// two. 1041 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { 1042 using namespace LegalityPredicates; 1043 return actionIf(LegalizeAction::MoreElements, 1044 numElementsNotPow2(typeIdx(TypeIdx)), 1045 LegalizeMutations::moreElementsToNextPow2(TypeIdx)); 1046 } 1047 1048 /// Limit the number of elements in EltTy vectors to at least MinElements. 1049 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy, 1050 unsigned MinElements) { 1051 // Mark the type index as covered: 1052 typeIdx(TypeIdx); 1053 return actionIf( 1054 LegalizeAction::MoreElements, 1055 [=](const LegalityQuery &Query) { 1056 LLT VecTy = Query.Types[TypeIdx]; 1057 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1058 VecTy.getNumElements() < MinElements; 1059 }, 1060 [=](const LegalityQuery &Query) { 1061 LLT VecTy = Query.Types[TypeIdx]; 1062 return std::make_pair( 1063 TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType())); 1064 }); 1065 } 1066 1067 /// Set number of elements to nearest larger multiple of NumElts. 1068 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy, 1069 unsigned NumElts) { 1070 typeIdx(TypeIdx); 1071 return actionIf( 1072 LegalizeAction::MoreElements, 1073 [=](const LegalityQuery &Query) { 1074 LLT VecTy = Query.Types[TypeIdx]; 1075 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1076 (VecTy.getNumElements() % NumElts != 0); 1077 }, 1078 [=](const LegalityQuery &Query) { 1079 LLT VecTy = Query.Types[TypeIdx]; 1080 unsigned NewSize = alignTo(VecTy.getNumElements(), NumElts); 1081 return std::make_pair( 1082 TypeIdx, LLT::fixed_vector(NewSize, VecTy.getElementType())); 1083 }); 1084 } 1085 1086 /// Limit the number of elements in EltTy vectors to at most MaxElements. 1087 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, 1088 unsigned MaxElements) { 1089 // Mark the type index as covered: 1090 typeIdx(TypeIdx); 1091 return actionIf( 1092 LegalizeAction::FewerElements, 1093 [=](const LegalityQuery &Query) { 1094 LLT VecTy = Query.Types[TypeIdx]; 1095 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1096 VecTy.getNumElements() > MaxElements; 1097 }, 1098 [=](const LegalityQuery &Query) { 1099 LLT VecTy = Query.Types[TypeIdx]; 1100 LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements), 1101 VecTy.getElementType()); 1102 return std::make_pair(TypeIdx, NewTy); 1103 }); 1104 } 1105 /// Limit the number of elements for the given vectors to at least MinTy's 1106 /// number of elements and at most MaxTy's number of elements. 1107 /// 1108 /// No effect if the type is not a vector or does not have the same element 1109 /// type as the constraints. 1110 /// The element type of MinTy and MaxTy must match. 1111 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy, 1112 const LLT MaxTy) { 1113 assert(MinTy.getElementType() == MaxTy.getElementType() && 1114 "Expected element types to agree"); 1115 1116 const LLT EltTy = MinTy.getElementType(); 1117 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) 1118 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); 1119 } 1120 1121 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements 1122 /// (or scalars when \p NumElts equals 1). 1123 /// First pad with undef elements to nearest larger multiple of \p NumElts. 1124 /// Then perform split with all sub-instructions having the same type. 1125 /// Using clampMaxNumElements (non-strict) can result in leftover instruction 1126 /// with different type (fewer elements then \p NumElts or scalar). 1127 /// No effect if the type is not a vector. 1128 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, 1129 unsigned NumElts) { 1130 return alignNumElementsTo(TypeIdx, EltTy, NumElts) 1131 .clampMaxNumElements(TypeIdx, EltTy, NumElts); 1132 } 1133 1134 /// Fallback on the previous implementation. This should only be used while 1135 /// porting a rule. 1136 LegalizeRuleSet &fallback() { 1137 add({always, LegalizeAction::UseLegacyRules}); 1138 return *this; 1139 } 1140 1141 /// Check if there is no type index which is obviously not handled by the 1142 /// LegalizeRuleSet in any way at all. 1143 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1144 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; 1145 /// Check if there is no imm index which is obviously not handled by the 1146 /// LegalizeRuleSet in any way at all. 1147 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1148 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; 1149 1150 /// Apply the ruleset to the given LegalityQuery. 1151 LegalizeActionStep apply(const LegalityQuery &Query) const; 1152 }; 1153 1154 class LegalizerInfo { 1155 public: 1156 virtual ~LegalizerInfo() = default; 1157 1158 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { 1159 return LegacyInfo; 1160 } 1161 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } 1162 1163 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; 1164 unsigned getActionDefinitionsIdx(unsigned Opcode) const; 1165 1166 /// Perform simple self-diagnostic and assert if there is anything obviously 1167 /// wrong with the actions set up. 1168 void verify(const MCInstrInfo &MII) const; 1169 1170 /// Get the action definitions for the given opcode. Use this to run a 1171 /// LegalityQuery through the definitions. 1172 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; 1173 1174 /// Get the action definition builder for the given opcode. Use this to define 1175 /// the action definitions. 1176 /// 1177 /// It is an error to request an opcode that has already been requested by the 1178 /// multiple-opcode variant. 1179 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); 1180 1181 /// Get the action definition builder for the given set of opcodes. Use this 1182 /// to define the action definitions for multiple opcodes at once. The first 1183 /// opcode given will be considered the representative opcode and will hold 1184 /// the definitions whereas the other opcodes will be configured to refer to 1185 /// the representative opcode. This lowers memory requirements and very 1186 /// slightly improves performance. 1187 /// 1188 /// It would be very easy to introduce unexpected side-effects as a result of 1189 /// this aliasing if it were permitted to request different but intersecting 1190 /// sets of opcodes but that is difficult to keep track of. It is therefore an 1191 /// error to request the same opcode twice using this API, to request an 1192 /// opcode that already has definitions, or to use the single-opcode API on an 1193 /// opcode that has already been requested by this API. 1194 LegalizeRuleSet & 1195 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); 1196 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); 1197 1198 /// Determine what action should be taken to legalize the described 1199 /// instruction. Requires computeTables to have been called. 1200 /// 1201 /// \returns a description of the next legalization step to perform. 1202 LegalizeActionStep getAction(const LegalityQuery &Query) const; 1203 1204 /// Determine what action should be taken to legalize the given generic 1205 /// instruction. 1206 /// 1207 /// \returns a description of the next legalization step to perform. 1208 LegalizeActionStep getAction(const MachineInstr &MI, 1209 const MachineRegisterInfo &MRI) const; 1210 1211 bool isLegal(const LegalityQuery &Query) const { 1212 return getAction(Query).Action == LegalizeAction::Legal; 1213 } 1214 1215 bool isLegalOrCustom(const LegalityQuery &Query) const { 1216 auto Action = getAction(Query).Action; 1217 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom; 1218 } 1219 1220 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; 1221 bool isLegalOrCustom(const MachineInstr &MI, 1222 const MachineRegisterInfo &MRI) const; 1223 1224 /// Called for instructions with the Custom LegalizationAction. 1225 virtual bool legalizeCustom(LegalizerHelper &Helper, 1226 MachineInstr &MI) const { 1227 llvm_unreachable("must implement this if custom action is used"); 1228 } 1229 1230 /// \returns true if MI is either legal or has been legalized and false if not 1231 /// legal. 1232 /// Return true if MI is either legal or has been legalized and false 1233 /// if not legal. 1234 virtual bool legalizeIntrinsic(LegalizerHelper &Helper, 1235 MachineInstr &MI) const { 1236 return true; 1237 } 1238 1239 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while 1240 /// widening a constant of type SmallTy which targets can override. 1241 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which 1242 /// will be the default. 1243 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; 1244 1245 private: 1246 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; 1247 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; 1248 1249 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; 1250 LegacyLegalizerInfo LegacyInfo; 1251 }; 1252 1253 #ifndef NDEBUG 1254 /// Checks that MIR is fully legal, returns an illegal instruction if it's not, 1255 /// nullptr otherwise 1256 const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); 1257 #endif 1258 1259 } // end namespace llvm. 1260 1261 #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 1262