1 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// Interface for Targets to specify which operations they can successfully 10 /// select and how the others should be expanded most efficiently. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 15 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 16 17 #include "llvm/ADT/SmallBitVector.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" 20 #include "llvm/CodeGen/MachineMemOperand.h" 21 #include "llvm/CodeGen/TargetOpcodes.h" 22 #include "llvm/MC/MCInstrDesc.h" 23 #include "llvm/Support/AtomicOrdering.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/LowLevelTypeImpl.h" 26 #include <cassert> 27 #include <cstdint> 28 #include <tuple> 29 #include <utility> 30 31 namespace llvm { 32 33 extern cl::opt<bool> DisableGISelLegalityCheck; 34 35 class MachineFunction; 36 class raw_ostream; 37 class LegalizerHelper; 38 class MachineInstr; 39 class MachineRegisterInfo; 40 class MCInstrInfo; 41 42 namespace LegalizeActions { 43 enum LegalizeAction : std::uint8_t { 44 /// The operation is expected to be selectable directly by the target, and 45 /// no transformation is necessary. 46 Legal, 47 48 /// The operation should be synthesized from multiple instructions acting on 49 /// a narrower scalar base-type. For example a 64-bit add might be 50 /// implemented in terms of 32-bit add-with-carry. 51 NarrowScalar, 52 53 /// The operation should be implemented in terms of a wider scalar 54 /// base-type. For example a <2 x s8> add could be implemented as a <2 55 /// x s32> add (ignoring the high bits). 56 WidenScalar, 57 58 /// The (vector) operation should be implemented by splitting it into 59 /// sub-vectors where the operation is legal. For example a <8 x s64> add 60 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover 61 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add 62 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover 63 /// types can be avoided by doing MoreElements first. 64 FewerElements, 65 66 /// The (vector) operation should be implemented by widening the input 67 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is 68 /// rarely legal, but you might perform an <8 x i8> and then only look at 69 /// the first two results. 70 MoreElements, 71 72 /// Perform the operation on a different, but equivalently sized type. 73 Bitcast, 74 75 /// The operation itself must be expressed in terms of simpler actions on 76 /// this target. E.g. a SREM replaced by an SDIV and subtraction. 77 Lower, 78 79 /// The operation should be implemented as a call to some kind of runtime 80 /// support library. For example this usually happens on machines that don't 81 /// support floating-point operations natively. 82 Libcall, 83 84 /// The target wants to do something special with this combination of 85 /// operand and type. A callback will be issued when it is needed. 86 Custom, 87 88 /// This operation is completely unsupported on the target. A programming 89 /// error has occurred. 90 Unsupported, 91 92 /// Sentinel value for when no action was found in the specified table. 93 NotFound, 94 95 /// Fall back onto the old rules. 96 /// TODO: Remove this once we've migrated 97 UseLegacyRules, 98 }; 99 } // end namespace LegalizeActions 100 raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); 101 102 using LegalizeActions::LegalizeAction; 103 104 /// The LegalityQuery object bundles together all the information that's needed 105 /// to decide whether a given operation is legal or not. 106 /// For efficiency, it doesn't make a copy of Types so care must be taken not 107 /// to free it before using the query. 108 struct LegalityQuery { 109 unsigned Opcode; 110 ArrayRef<LLT> Types; 111 112 struct MemDesc { 113 LLT MemoryTy; 114 uint64_t AlignInBits; 115 AtomicOrdering Ordering; 116 117 MemDesc() = default; 118 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering) 119 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {} 120 MemDesc(const MachineMemOperand &MMO) 121 : MemoryTy(MMO.getMemoryType()), 122 AlignInBits(MMO.getAlign().value() * 8), 123 Ordering(MMO.getSuccessOrdering()) {} 124 }; 125 126 /// Operations which require memory can use this to place requirements on the 127 /// memory type for each MMO. 128 ArrayRef<MemDesc> MMODescrs; 129 130 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, 131 const ArrayRef<MemDesc> MMODescrs) 132 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} 133 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) 134 : LegalityQuery(Opcode, Types, {}) {} 135 136 raw_ostream &print(raw_ostream &OS) const; 137 }; 138 139 /// The result of a query. It either indicates a final answer of Legal or 140 /// Unsupported or describes an action that must be taken to make an operation 141 /// more legal. 142 struct LegalizeActionStep { 143 /// The action to take or the final answer. 144 LegalizeAction Action; 145 /// If describing an action, the type index to change. Otherwise zero. 146 unsigned TypeIdx; 147 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. 148 LLT NewType; 149 150 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, 151 const LLT NewType) 152 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} 153 154 LegalizeActionStep(LegacyLegalizeActionStep Step) 155 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { 156 switch (Step.Action) { 157 case LegacyLegalizeActions::Legal: 158 Action = LegalizeActions::Legal; 159 break; 160 case LegacyLegalizeActions::NarrowScalar: 161 Action = LegalizeActions::NarrowScalar; 162 break; 163 case LegacyLegalizeActions::WidenScalar: 164 Action = LegalizeActions::WidenScalar; 165 break; 166 case LegacyLegalizeActions::FewerElements: 167 Action = LegalizeActions::FewerElements; 168 break; 169 case LegacyLegalizeActions::MoreElements: 170 Action = LegalizeActions::MoreElements; 171 break; 172 case LegacyLegalizeActions::Bitcast: 173 Action = LegalizeActions::Bitcast; 174 break; 175 case LegacyLegalizeActions::Lower: 176 Action = LegalizeActions::Lower; 177 break; 178 case LegacyLegalizeActions::Libcall: 179 Action = LegalizeActions::Libcall; 180 break; 181 case LegacyLegalizeActions::Custom: 182 Action = LegalizeActions::Custom; 183 break; 184 case LegacyLegalizeActions::Unsupported: 185 Action = LegalizeActions::Unsupported; 186 break; 187 case LegacyLegalizeActions::NotFound: 188 Action = LegalizeActions::NotFound; 189 break; 190 } 191 } 192 193 bool operator==(const LegalizeActionStep &RHS) const { 194 return std::tie(Action, TypeIdx, NewType) == 195 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); 196 } 197 }; 198 199 using LegalityPredicate = std::function<bool (const LegalityQuery &)>; 200 using LegalizeMutation = 201 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; 202 203 namespace LegalityPredicates { 204 struct TypePairAndMemDesc { 205 LLT Type0; 206 LLT Type1; 207 LLT MemTy; 208 uint64_t Align; 209 210 bool operator==(const TypePairAndMemDesc &Other) const { 211 return Type0 == Other.Type0 && Type1 == Other.Type1 && 212 Align == Other.Align && MemTy == Other.MemTy; 213 } 214 215 /// \returns true if this memory access is legal with for the access described 216 /// by \p Other (The alignment is sufficient for the size and result type). 217 bool isCompatible(const TypePairAndMemDesc &Other) const { 218 return Type0 == Other.Type0 && Type1 == Other.Type1 && 219 Align >= Other.Align && 220 // FIXME: This perhaps should be stricter, but the current legality 221 // rules are written only considering the size. 222 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); 223 } 224 }; 225 226 /// True iff P0 and P1 are true. 227 template<typename Predicate> 228 Predicate all(Predicate P0, Predicate P1) { 229 return [=](const LegalityQuery &Query) { 230 return P0(Query) && P1(Query); 231 }; 232 } 233 /// True iff all given predicates are true. 234 template<typename Predicate, typename... Args> 235 Predicate all(Predicate P0, Predicate P1, Args... args) { 236 return all(all(P0, P1), args...); 237 } 238 239 /// True iff P0 or P1 are true. 240 template<typename Predicate> 241 Predicate any(Predicate P0, Predicate P1) { 242 return [=](const LegalityQuery &Query) { 243 return P0(Query) || P1(Query); 244 }; 245 } 246 /// True iff any given predicates are true. 247 template<typename Predicate, typename... Args> 248 Predicate any(Predicate P0, Predicate P1, Args... args) { 249 return any(any(P0, P1), args...); 250 } 251 252 /// True iff the given type index is the specified type. 253 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); 254 /// True iff the given type index is one of the specified types. 255 LegalityPredicate typeInSet(unsigned TypeIdx, 256 std::initializer_list<LLT> TypesInit); 257 258 /// True iff the given type index is not the specified type. 259 inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { 260 return [=](const LegalityQuery &Query) { 261 return Query.Types[TypeIdx] != Type; 262 }; 263 } 264 265 /// True iff the given types for the given pair of type indexes is one of the 266 /// specified type pairs. 267 LegalityPredicate 268 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, 269 std::initializer_list<std::pair<LLT, LLT>> TypesInit); 270 /// True iff the given types for the given pair of type indexes is one of the 271 /// specified type pairs. 272 LegalityPredicate typePairAndMemDescInSet( 273 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, 274 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); 275 /// True iff the specified type index is a scalar. 276 LegalityPredicate isScalar(unsigned TypeIdx); 277 /// True iff the specified type index is a vector. 278 LegalityPredicate isVector(unsigned TypeIdx); 279 /// True iff the specified type index is a pointer (with any address space). 280 LegalityPredicate isPointer(unsigned TypeIdx); 281 /// True iff the specified type index is a pointer with the specified address 282 /// space. 283 LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); 284 285 /// True if the type index is a vector with element type \p EltTy 286 LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy); 287 288 /// True iff the specified type index is a scalar that's narrower than the given 289 /// size. 290 LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size); 291 292 /// True iff the specified type index is a scalar that's wider than the given 293 /// size. 294 LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size); 295 296 /// True iff the specified type index is a scalar or vector with an element type 297 /// that's narrower than the given size. 298 LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); 299 300 /// True iff the specified type index is a scalar or a vector with an element 301 /// type that's wider than the given size. 302 LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); 303 304 /// True iff the specified type index is a scalar whose size is not a multiple 305 /// of Size. 306 LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size); 307 308 /// True iff the specified type index is a scalar whose size is not a power of 309 /// 2. 310 LegalityPredicate sizeNotPow2(unsigned TypeIdx); 311 312 /// True iff the specified type index is a scalar or vector whose element size 313 /// is not a power of 2. 314 LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); 315 316 /// True if the total bitwidth of the specified type index is \p Size bits. 317 LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size); 318 319 /// True iff the specified type indices are both the same bit size. 320 LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); 321 322 /// True iff the first type index has a larger total bit size than second type 323 /// index. 324 LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); 325 326 /// True iff the first type index has a smaller total bit size than second type 327 /// index. 328 LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); 329 330 /// True iff the specified MMO index has a size (rounded to bytes) that is not a 331 /// power of 2. 332 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); 333 334 /// True iff the specified MMO index has a size that is not an even byte size, 335 /// or that even byte size is not a power of 2. 336 LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx); 337 338 /// True iff the specified type index is a vector whose element count is not a 339 /// power of 2. 340 LegalityPredicate numElementsNotPow2(unsigned TypeIdx); 341 /// True iff the specified MMO index has at an atomic ordering of at Ordering or 342 /// stronger. 343 LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, 344 AtomicOrdering Ordering); 345 } // end namespace LegalityPredicates 346 347 namespace LegalizeMutations { 348 /// Select this specific type for the given type index. 349 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); 350 351 /// Keep the same type as the given type index. 352 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); 353 354 /// Keep the same scalar or element type as the given type index. 355 LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); 356 357 /// Keep the same scalar or element type as the given type. 358 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); 359 360 /// Keep the same scalar or element type as \p TypeIdx, but take the number of 361 /// elements from \p FromTypeIdx. 362 LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx); 363 364 /// Keep the same scalar or element type as \p TypeIdx, but take the number of 365 /// elements from \p Ty. 366 LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty); 367 368 /// Change the scalar size or element size to have the same scalar size as type 369 /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and 370 /// only changes the size. 371 LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx); 372 373 /// Widen the scalar type or vector element type for the given type index to the 374 /// next power of 2. 375 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); 376 377 /// Widen the scalar type or vector element type for the given type index to 378 /// next multiple of \p Size. 379 LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx, 380 unsigned Size); 381 382 /// Add more elements to the type for the given type index to the next power of 383 /// 2. 384 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); 385 /// Break up the vector type for the given type index into the element type. 386 LegalizeMutation scalarize(unsigned TypeIdx); 387 } // end namespace LegalizeMutations 388 389 /// A single rule in a legalizer info ruleset. 390 /// The specified action is chosen when the predicate is true. Where appropriate 391 /// for the action (e.g. for WidenScalar) the new type is selected using the 392 /// given mutator. 393 class LegalizeRule { 394 LegalityPredicate Predicate; 395 LegalizeAction Action; 396 LegalizeMutation Mutation; 397 398 public: 399 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, 400 LegalizeMutation Mutation = nullptr) 401 : Predicate(Predicate), Action(Action), Mutation(Mutation) {} 402 403 /// Test whether the LegalityQuery matches. 404 bool match(const LegalityQuery &Query) const { 405 return Predicate(Query); 406 } 407 408 LegalizeAction getAction() const { return Action; } 409 410 /// Determine the change to make. 411 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { 412 if (Mutation) 413 return Mutation(Query); 414 return std::make_pair(0, LLT{}); 415 } 416 }; 417 418 class LegalizeRuleSet { 419 /// When non-zero, the opcode we are an alias of 420 unsigned AliasOf = 0; 421 /// If true, there is another opcode that aliases this one 422 bool IsAliasedByAnother = false; 423 SmallVector<LegalizeRule, 2> Rules; 424 425 #ifndef NDEBUG 426 /// If bit I is set, this rule set contains a rule that may handle (predicate 427 /// or perform an action upon (or both)) the type index I. The uncertainty 428 /// comes from free-form rules executing user-provided lambda functions. We 429 /// conservatively assume such rules do the right thing and cover all type 430 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs 431 /// to be to distinguish such cases from the cases where all type indices are 432 /// individually handled. 433 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - 434 MCOI::OPERAND_FIRST_GENERIC + 2}; 435 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - 436 MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; 437 #endif 438 439 unsigned typeIdx(unsigned TypeIdx) { 440 assert(TypeIdx <= 441 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && 442 "Type Index is out of bounds"); 443 #ifndef NDEBUG 444 TypeIdxsCovered.set(TypeIdx); 445 #endif 446 return TypeIdx; 447 } 448 449 void markAllIdxsAsCovered() { 450 #ifndef NDEBUG 451 TypeIdxsCovered.set(); 452 ImmIdxsCovered.set(); 453 #endif 454 } 455 456 void add(const LegalizeRule &Rule) { 457 assert(AliasOf == 0 && 458 "RuleSet is aliased, change the representative opcode instead"); 459 Rules.push_back(Rule); 460 } 461 462 static bool always(const LegalityQuery &) { return true; } 463 464 /// Use the given action when the predicate is true. 465 /// Action should not be an action that requires mutation. 466 LegalizeRuleSet &actionIf(LegalizeAction Action, 467 LegalityPredicate Predicate) { 468 add({Predicate, Action}); 469 return *this; 470 } 471 /// Use the given action when the predicate is true. 472 /// Action should be an action that requires mutation. 473 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, 474 LegalizeMutation Mutation) { 475 add({Predicate, Action, Mutation}); 476 return *this; 477 } 478 /// Use the given action when type index 0 is any type in the given list. 479 /// Action should not be an action that requires mutation. 480 LegalizeRuleSet &actionFor(LegalizeAction Action, 481 std::initializer_list<LLT> Types) { 482 using namespace LegalityPredicates; 483 return actionIf(Action, typeInSet(typeIdx(0), Types)); 484 } 485 /// Use the given action when type index 0 is any type in the given list. 486 /// Action should be an action that requires mutation. 487 LegalizeRuleSet &actionFor(LegalizeAction Action, 488 std::initializer_list<LLT> Types, 489 LegalizeMutation Mutation) { 490 using namespace LegalityPredicates; 491 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation); 492 } 493 /// Use the given action when type indexes 0 and 1 is any type pair in the 494 /// given list. 495 /// Action should not be an action that requires mutation. 496 LegalizeRuleSet &actionFor(LegalizeAction Action, 497 std::initializer_list<std::pair<LLT, LLT>> Types) { 498 using namespace LegalityPredicates; 499 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 500 } 501 /// Use the given action when type indexes 0 and 1 is any type pair in the 502 /// given list. 503 /// Action should be an action that requires mutation. 504 LegalizeRuleSet &actionFor(LegalizeAction Action, 505 std::initializer_list<std::pair<LLT, LLT>> Types, 506 LegalizeMutation Mutation) { 507 using namespace LegalityPredicates; 508 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), 509 Mutation); 510 } 511 /// Use the given action when type index 0 is any type in the given list and 512 /// imm index 0 is anything. Action should not be an action that requires 513 /// mutation. 514 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, 515 std::initializer_list<LLT> Types) { 516 using namespace LegalityPredicates; 517 immIdx(0); // Inform verifier imm idx 0 is handled. 518 return actionIf(Action, typeInSet(typeIdx(0), Types)); 519 } 520 521 LegalizeRuleSet &actionForTypeWithAnyImm( 522 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) { 523 using namespace LegalityPredicates; 524 immIdx(0); // Inform verifier imm idx 0 is handled. 525 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 526 } 527 528 /// Use the given action when type indexes 0 and 1 are both in the given list. 529 /// That is, the type pair is in the cartesian product of the list. 530 /// Action should not be an action that requires mutation. 531 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, 532 std::initializer_list<LLT> Types) { 533 using namespace LegalityPredicates; 534 return actionIf(Action, all(typeInSet(typeIdx(0), Types), 535 typeInSet(typeIdx(1), Types))); 536 } 537 /// Use the given action when type indexes 0 and 1 are both in their 538 /// respective lists. 539 /// That is, the type pair is in the cartesian product of the lists 540 /// Action should not be an action that requires mutation. 541 LegalizeRuleSet & 542 actionForCartesianProduct(LegalizeAction Action, 543 std::initializer_list<LLT> Types0, 544 std::initializer_list<LLT> Types1) { 545 using namespace LegalityPredicates; 546 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 547 typeInSet(typeIdx(1), Types1))); 548 } 549 /// Use the given action when type indexes 0, 1, and 2 are all in their 550 /// respective lists. 551 /// That is, the type triple is in the cartesian product of the lists 552 /// Action should not be an action that requires mutation. 553 LegalizeRuleSet &actionForCartesianProduct( 554 LegalizeAction Action, std::initializer_list<LLT> Types0, 555 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { 556 using namespace LegalityPredicates; 557 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 558 all(typeInSet(typeIdx(1), Types1), 559 typeInSet(typeIdx(2), Types2)))); 560 } 561 562 public: 563 LegalizeRuleSet() = default; 564 565 bool isAliasedByAnother() { return IsAliasedByAnother; } 566 void setIsAliasedByAnother() { IsAliasedByAnother = true; } 567 void aliasTo(unsigned Opcode) { 568 assert((AliasOf == 0 || AliasOf == Opcode) && 569 "Opcode is already aliased to another opcode"); 570 assert(Rules.empty() && "Aliasing will discard rules"); 571 AliasOf = Opcode; 572 } 573 unsigned getAlias() const { return AliasOf; } 574 575 unsigned immIdx(unsigned ImmIdx) { 576 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - 577 MCOI::OPERAND_FIRST_GENERIC_IMM) && 578 "Imm Index is out of bounds"); 579 #ifndef NDEBUG 580 ImmIdxsCovered.set(ImmIdx); 581 #endif 582 return ImmIdx; 583 } 584 585 /// The instruction is legal if predicate is true. 586 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { 587 // We have no choice but conservatively assume that the free-form 588 // user-provided Predicate properly handles all type indices: 589 markAllIdxsAsCovered(); 590 return actionIf(LegalizeAction::Legal, Predicate); 591 } 592 /// The instruction is legal when type index 0 is any type in the given list. 593 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { 594 return actionFor(LegalizeAction::Legal, Types); 595 } 596 /// The instruction is legal when type indexes 0 and 1 is any type pair in the 597 /// given list. 598 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 599 return actionFor(LegalizeAction::Legal, Types); 600 } 601 /// The instruction is legal when type index 0 is any type in the given list 602 /// and imm index 0 is anything. 603 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { 604 markAllIdxsAsCovered(); 605 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 606 } 607 608 LegalizeRuleSet &legalForTypeWithAnyImm( 609 std::initializer_list<std::pair<LLT, LLT>> Types) { 610 markAllIdxsAsCovered(); 611 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 612 } 613 614 /// The instruction is legal when type indexes 0 and 1 along with the memory 615 /// size and minimum alignment is any type and size tuple in the given list. 616 LegalizeRuleSet &legalForTypesWithMemDesc( 617 std::initializer_list<LegalityPredicates::TypePairAndMemDesc> 618 TypesAndMemDesc) { 619 return actionIf(LegalizeAction::Legal, 620 LegalityPredicates::typePairAndMemDescInSet( 621 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); 622 } 623 /// The instruction is legal when type indexes 0 and 1 are both in the given 624 /// list. That is, the type pair is in the cartesian product of the list. 625 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { 626 return actionForCartesianProduct(LegalizeAction::Legal, Types); 627 } 628 /// The instruction is legal when type indexes 0 and 1 are both their 629 /// respective lists. 630 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 631 std::initializer_list<LLT> Types1) { 632 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); 633 } 634 /// The instruction is legal when type indexes 0, 1, and 2 are both their 635 /// respective lists. 636 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 637 std::initializer_list<LLT> Types1, 638 std::initializer_list<LLT> Types2) { 639 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, 640 Types2); 641 } 642 643 LegalizeRuleSet &alwaysLegal() { 644 using namespace LegalizeMutations; 645 markAllIdxsAsCovered(); 646 return actionIf(LegalizeAction::Legal, always); 647 } 648 649 /// The specified type index is coerced if predicate is true. 650 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, 651 LegalizeMutation Mutation) { 652 // We have no choice but conservatively assume that lowering with a 653 // free-form user provided Predicate properly handles all type indices: 654 markAllIdxsAsCovered(); 655 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation); 656 } 657 658 /// The instruction is lowered. 659 LegalizeRuleSet &lower() { 660 using namespace LegalizeMutations; 661 // We have no choice but conservatively assume that predicate-less lowering 662 // properly handles all type indices by design: 663 markAllIdxsAsCovered(); 664 return actionIf(LegalizeAction::Lower, always); 665 } 666 /// The instruction is lowered if predicate is true. Keep type index 0 as the 667 /// same type. 668 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { 669 using namespace LegalizeMutations; 670 // We have no choice but conservatively assume that lowering with a 671 // free-form user provided Predicate properly handles all type indices: 672 markAllIdxsAsCovered(); 673 return actionIf(LegalizeAction::Lower, Predicate); 674 } 675 /// The instruction is lowered if predicate is true. 676 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, 677 LegalizeMutation Mutation) { 678 // We have no choice but conservatively assume that lowering with a 679 // free-form user provided Predicate properly handles all type indices: 680 markAllIdxsAsCovered(); 681 return actionIf(LegalizeAction::Lower, Predicate, Mutation); 682 } 683 /// The instruction is lowered when type index 0 is any type in the given 684 /// list. Keep type index 0 as the same type. 685 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { 686 return actionFor(LegalizeAction::Lower, Types); 687 } 688 /// The instruction is lowered when type index 0 is any type in the given 689 /// list. 690 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, 691 LegalizeMutation Mutation) { 692 return actionFor(LegalizeAction::Lower, Types, Mutation); 693 } 694 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 695 /// the given list. Keep type index 0 as the same type. 696 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 697 return actionFor(LegalizeAction::Lower, Types); 698 } 699 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 700 /// the given list. 701 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, 702 LegalizeMutation Mutation) { 703 return actionFor(LegalizeAction::Lower, Types, Mutation); 704 } 705 /// The instruction is lowered when type indexes 0 and 1 are both in their 706 /// respective lists. 707 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 708 std::initializer_list<LLT> Types1) { 709 using namespace LegalityPredicates; 710 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1); 711 } 712 /// The instruction is lowered when when type indexes 0, 1, and 2 are all in 713 /// their respective lists. 714 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 715 std::initializer_list<LLT> Types1, 716 std::initializer_list<LLT> Types2) { 717 using namespace LegalityPredicates; 718 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1, 719 Types2); 720 } 721 722 /// The instruction is emitted as a library call. 723 LegalizeRuleSet &libcall() { 724 using namespace LegalizeMutations; 725 // We have no choice but conservatively assume that predicate-less lowering 726 // properly handles all type indices by design: 727 markAllIdxsAsCovered(); 728 return actionIf(LegalizeAction::Libcall, always); 729 } 730 731 /// Like legalIf, but for the Libcall action. 732 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { 733 // We have no choice but conservatively assume that a libcall with a 734 // free-form user provided Predicate properly handles all type indices: 735 markAllIdxsAsCovered(); 736 return actionIf(LegalizeAction::Libcall, Predicate); 737 } 738 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { 739 return actionFor(LegalizeAction::Libcall, Types); 740 } 741 LegalizeRuleSet & 742 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 743 return actionFor(LegalizeAction::Libcall, Types); 744 } 745 LegalizeRuleSet & 746 libcallForCartesianProduct(std::initializer_list<LLT> Types) { 747 return actionForCartesianProduct(LegalizeAction::Libcall, Types); 748 } 749 LegalizeRuleSet & 750 libcallForCartesianProduct(std::initializer_list<LLT> Types0, 751 std::initializer_list<LLT> Types1) { 752 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); 753 } 754 755 /// Widen the scalar to the one selected by the mutation if the predicate is 756 /// true. 757 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, 758 LegalizeMutation Mutation) { 759 // We have no choice but conservatively assume that an action with a 760 // free-form user provided Predicate properly handles all type indices: 761 markAllIdxsAsCovered(); 762 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); 763 } 764 /// Narrow the scalar to the one selected by the mutation if the predicate is 765 /// true. 766 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, 767 LegalizeMutation Mutation) { 768 // We have no choice but conservatively assume that an action with a 769 // free-form user provided Predicate properly handles all type indices: 770 markAllIdxsAsCovered(); 771 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); 772 } 773 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any 774 /// type pair in the given list. 775 LegalizeRuleSet & 776 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types, 777 LegalizeMutation Mutation) { 778 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation); 779 } 780 781 /// Add more elements to reach the type selected by the mutation if the 782 /// predicate is true. 783 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, 784 LegalizeMutation Mutation) { 785 // We have no choice but conservatively assume that an action with a 786 // free-form user provided Predicate properly handles all type indices: 787 markAllIdxsAsCovered(); 788 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); 789 } 790 /// Remove elements to reach the type selected by the mutation if the 791 /// predicate is true. 792 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, 793 LegalizeMutation Mutation) { 794 // We have no choice but conservatively assume that an action with a 795 // free-form user provided Predicate properly handles all type indices: 796 markAllIdxsAsCovered(); 797 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); 798 } 799 800 /// The instruction is unsupported. 801 LegalizeRuleSet &unsupported() { 802 markAllIdxsAsCovered(); 803 return actionIf(LegalizeAction::Unsupported, always); 804 } 805 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { 806 return actionIf(LegalizeAction::Unsupported, Predicate); 807 } 808 809 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) { 810 return actionFor(LegalizeAction::Unsupported, Types); 811 } 812 813 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { 814 return actionIf(LegalizeAction::Unsupported, 815 LegalityPredicates::memSizeInBytesNotPow2(0)); 816 } 817 818 /// Lower a memory operation if the memory size, rounded to bytes, is not a 819 /// power of 2. For example, this will not trigger for s1 or s7, but will for 820 /// s24. 821 LegalizeRuleSet &lowerIfMemSizeNotPow2() { 822 return actionIf(LegalizeAction::Lower, 823 LegalityPredicates::memSizeInBytesNotPow2(0)); 824 } 825 826 /// Lower a memory operation if the memory access size is not a round power of 827 /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely 828 /// what you want (e.g. this will lower s1, s7 and s24). 829 LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() { 830 return actionIf(LegalizeAction::Lower, 831 LegalityPredicates::memSizeNotByteSizePow2(0)); 832 } 833 834 LegalizeRuleSet &customIf(LegalityPredicate Predicate) { 835 // We have no choice but conservatively assume that a custom action with a 836 // free-form user provided Predicate properly handles all type indices: 837 markAllIdxsAsCovered(); 838 return actionIf(LegalizeAction::Custom, Predicate); 839 } 840 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { 841 return actionFor(LegalizeAction::Custom, Types); 842 } 843 844 /// The instruction is custom when type indexes 0 and 1 is any type pair in the 845 /// given list. 846 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 847 return actionFor(LegalizeAction::Custom, Types); 848 } 849 850 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { 851 return actionForCartesianProduct(LegalizeAction::Custom, Types); 852 } 853 /// The instruction is custom when type indexes 0 and 1 are both in their 854 /// respective lists. 855 LegalizeRuleSet & 856 customForCartesianProduct(std::initializer_list<LLT> Types0, 857 std::initializer_list<LLT> Types1) { 858 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); 859 } 860 /// The instruction is custom when when type indexes 0, 1, and 2 are all in 861 /// their respective lists. 862 LegalizeRuleSet & 863 customForCartesianProduct(std::initializer_list<LLT> Types0, 864 std::initializer_list<LLT> Types1, 865 std::initializer_list<LLT> Types2) { 866 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1, 867 Types2); 868 } 869 870 /// Unconditionally custom lower. 871 LegalizeRuleSet &custom() { 872 return customIf(always); 873 } 874 875 /// Widen the scalar to the next power of two that is at least MinSize. 876 /// No effect if the type is not a scalar or is a power of two. 877 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, 878 unsigned MinSize = 0) { 879 using namespace LegalityPredicates; 880 return actionIf( 881 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), 882 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 883 } 884 885 /// Widen the scalar to the next multiple of Size. No effect if the 886 /// type is not a scalar or is a multiple of Size. 887 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx, 888 unsigned Size) { 889 using namespace LegalityPredicates; 890 return actionIf( 891 LegalizeAction::WidenScalar, sizeNotMultipleOf(typeIdx(TypeIdx), Size), 892 LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size)); 893 } 894 895 /// Widen the scalar or vector element type to the next power of two that is 896 /// at least MinSize. No effect if the scalar size is a power of two. 897 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, 898 unsigned MinSize = 0) { 899 using namespace LegalityPredicates; 900 return actionIf( 901 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), 902 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 903 } 904 905 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { 906 using namespace LegalityPredicates; 907 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)), 908 Mutation); 909 } 910 911 LegalizeRuleSet &scalarize(unsigned TypeIdx) { 912 using namespace LegalityPredicates; 913 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), 914 LegalizeMutations::scalarize(TypeIdx)); 915 } 916 917 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) { 918 using namespace LegalityPredicates; 919 return actionIf(LegalizeAction::FewerElements, 920 all(Predicate, isVector(typeIdx(TypeIdx))), 921 LegalizeMutations::scalarize(TypeIdx)); 922 } 923 924 /// Ensure the scalar or element is at least as wide as Ty. 925 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) { 926 using namespace LegalityPredicates; 927 using namespace LegalizeMutations; 928 return actionIf(LegalizeAction::WidenScalar, 929 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), 930 changeElementTo(typeIdx(TypeIdx), Ty)); 931 } 932 933 /// Ensure the scalar or element is at least as wide as Ty. 934 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, 935 unsigned TypeIdx, const LLT Ty) { 936 using namespace LegalityPredicates; 937 using namespace LegalizeMutations; 938 return actionIf(LegalizeAction::WidenScalar, 939 all(Predicate, scalarOrEltNarrowerThan( 940 TypeIdx, Ty.getScalarSizeInBits())), 941 changeElementTo(typeIdx(TypeIdx), Ty)); 942 } 943 944 /// Ensure the scalar is at least as wide as Ty. 945 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) { 946 using namespace LegalityPredicates; 947 using namespace LegalizeMutations; 948 return actionIf(LegalizeAction::WidenScalar, 949 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()), 950 changeTo(typeIdx(TypeIdx), Ty)); 951 } 952 953 /// Ensure the scalar is at least as wide as Ty if condition is met. 954 LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 955 const LLT Ty) { 956 using namespace LegalityPredicates; 957 using namespace LegalizeMutations; 958 return actionIf( 959 LegalizeAction::WidenScalar, 960 [=](const LegalityQuery &Query) { 961 const LLT QueryTy = Query.Types[TypeIdx]; 962 return QueryTy.isScalar() && 963 QueryTy.getSizeInBits() < Ty.getSizeInBits() && 964 Predicate(Query); 965 }, 966 changeTo(typeIdx(TypeIdx), Ty)); 967 } 968 969 /// Ensure the scalar is at most as wide as Ty. 970 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) { 971 using namespace LegalityPredicates; 972 using namespace LegalizeMutations; 973 return actionIf(LegalizeAction::NarrowScalar, 974 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), 975 changeElementTo(typeIdx(TypeIdx), Ty)); 976 } 977 978 /// Ensure the scalar is at most as wide as Ty. 979 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) { 980 using namespace LegalityPredicates; 981 using namespace LegalizeMutations; 982 return actionIf(LegalizeAction::NarrowScalar, 983 scalarWiderThan(TypeIdx, Ty.getSizeInBits()), 984 changeTo(typeIdx(TypeIdx), Ty)); 985 } 986 987 /// Conditionally limit the maximum size of the scalar. 988 /// For example, when the maximum size of one type depends on the size of 989 /// another such as extracting N bits from an M bit container. 990 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 991 const LLT Ty) { 992 using namespace LegalityPredicates; 993 using namespace LegalizeMutations; 994 return actionIf( 995 LegalizeAction::NarrowScalar, 996 [=](const LegalityQuery &Query) { 997 const LLT QueryTy = Query.Types[TypeIdx]; 998 return QueryTy.isScalar() && 999 QueryTy.getSizeInBits() > Ty.getSizeInBits() && 1000 Predicate(Query); 1001 }, 1002 changeElementTo(typeIdx(TypeIdx), Ty)); 1003 } 1004 1005 /// Limit the range of scalar sizes to MinTy and MaxTy. 1006 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy, 1007 const LLT MaxTy) { 1008 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); 1009 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); 1010 } 1011 1012 /// Limit the range of scalar sizes to MinTy and MaxTy. 1013 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, 1014 const LLT MaxTy) { 1015 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); 1016 } 1017 1018 /// Widen the scalar to match the size of another. 1019 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { 1020 typeIdx(TypeIdx); 1021 return widenScalarIf( 1022 [=](const LegalityQuery &Query) { 1023 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1024 Query.Types[TypeIdx].getSizeInBits(); 1025 }, 1026 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx)); 1027 } 1028 1029 /// Narrow the scalar to match the size of another. 1030 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) { 1031 typeIdx(TypeIdx); 1032 return narrowScalarIf( 1033 [=](const LegalityQuery &Query) { 1034 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() < 1035 Query.Types[TypeIdx].getSizeInBits(); 1036 }, 1037 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx)); 1038 } 1039 1040 /// Change the type \p TypeIdx to have the same scalar size as type \p 1041 /// SameSizeIdx. 1042 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) { 1043 return minScalarSameAs(TypeIdx, SameSizeIdx) 1044 .maxScalarSameAs(TypeIdx, SameSizeIdx); 1045 } 1046 1047 /// Conditionally widen the scalar or elt to match the size of another. 1048 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, 1049 unsigned TypeIdx, unsigned LargeTypeIdx) { 1050 typeIdx(TypeIdx); 1051 return widenScalarIf( 1052 [=](const LegalityQuery &Query) { 1053 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1054 Query.Types[TypeIdx].getScalarSizeInBits() && 1055 Predicate(Query); 1056 }, 1057 [=](const LegalityQuery &Query) { 1058 LLT T = Query.Types[LargeTypeIdx]; 1059 if (T.isVector() && T.getElementType().isPointer()) 1060 T = T.changeElementType(LLT::scalar(T.getScalarSizeInBits())); 1061 return std::make_pair(TypeIdx, T); 1062 }); 1063 } 1064 1065 /// Conditionally narrow the scalar or elt to match the size of another. 1066 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, 1067 unsigned TypeIdx, 1068 unsigned SmallTypeIdx) { 1069 typeIdx(TypeIdx); 1070 return narrowScalarIf( 1071 [=](const LegalityQuery &Query) { 1072 return Query.Types[SmallTypeIdx].getScalarSizeInBits() < 1073 Query.Types[TypeIdx].getScalarSizeInBits() && 1074 Predicate(Query); 1075 }, 1076 [=](const LegalityQuery &Query) { 1077 LLT T = Query.Types[SmallTypeIdx]; 1078 return std::make_pair(TypeIdx, T); 1079 }); 1080 } 1081 1082 /// Add more elements to the vector to reach the next power of two. 1083 /// No effect if the type is not a vector or the element count is a power of 1084 /// two. 1085 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { 1086 using namespace LegalityPredicates; 1087 return actionIf(LegalizeAction::MoreElements, 1088 numElementsNotPow2(typeIdx(TypeIdx)), 1089 LegalizeMutations::moreElementsToNextPow2(TypeIdx)); 1090 } 1091 1092 /// Limit the number of elements in EltTy vectors to at least MinElements. 1093 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy, 1094 unsigned MinElements) { 1095 // Mark the type index as covered: 1096 typeIdx(TypeIdx); 1097 return actionIf( 1098 LegalizeAction::MoreElements, 1099 [=](const LegalityQuery &Query) { 1100 LLT VecTy = Query.Types[TypeIdx]; 1101 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1102 VecTy.getNumElements() < MinElements; 1103 }, 1104 [=](const LegalityQuery &Query) { 1105 LLT VecTy = Query.Types[TypeIdx]; 1106 return std::make_pair( 1107 TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType())); 1108 }); 1109 } 1110 1111 /// Set number of elements to nearest larger multiple of NumElts. 1112 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy, 1113 unsigned NumElts) { 1114 typeIdx(TypeIdx); 1115 return actionIf( 1116 LegalizeAction::MoreElements, 1117 [=](const LegalityQuery &Query) { 1118 LLT VecTy = Query.Types[TypeIdx]; 1119 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1120 (VecTy.getNumElements() % NumElts != 0); 1121 }, 1122 [=](const LegalityQuery &Query) { 1123 LLT VecTy = Query.Types[TypeIdx]; 1124 unsigned NewSize = alignTo(VecTy.getNumElements(), NumElts); 1125 return std::make_pair( 1126 TypeIdx, LLT::fixed_vector(NewSize, VecTy.getElementType())); 1127 }); 1128 } 1129 1130 /// Limit the number of elements in EltTy vectors to at most MaxElements. 1131 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, 1132 unsigned MaxElements) { 1133 // Mark the type index as covered: 1134 typeIdx(TypeIdx); 1135 return actionIf( 1136 LegalizeAction::FewerElements, 1137 [=](const LegalityQuery &Query) { 1138 LLT VecTy = Query.Types[TypeIdx]; 1139 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1140 VecTy.getNumElements() > MaxElements; 1141 }, 1142 [=](const LegalityQuery &Query) { 1143 LLT VecTy = Query.Types[TypeIdx]; 1144 LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements), 1145 VecTy.getElementType()); 1146 return std::make_pair(TypeIdx, NewTy); 1147 }); 1148 } 1149 /// Limit the number of elements for the given vectors to at least MinTy's 1150 /// number of elements and at most MaxTy's number of elements. 1151 /// 1152 /// No effect if the type is not a vector or does not have the same element 1153 /// type as the constraints. 1154 /// The element type of MinTy and MaxTy must match. 1155 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy, 1156 const LLT MaxTy) { 1157 assert(MinTy.getElementType() == MaxTy.getElementType() && 1158 "Expected element types to agree"); 1159 1160 const LLT EltTy = MinTy.getElementType(); 1161 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) 1162 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); 1163 } 1164 1165 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements 1166 /// (or scalars when \p NumElts equals 1). 1167 /// First pad with undef elements to nearest larger multiple of \p NumElts. 1168 /// Then perform split with all sub-instructions having the same type. 1169 /// Using clampMaxNumElements (non-strict) can result in leftover instruction 1170 /// with different type (fewer elements then \p NumElts or scalar). 1171 /// No effect if the type is not a vector. 1172 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, 1173 unsigned NumElts) { 1174 return alignNumElementsTo(TypeIdx, EltTy, NumElts) 1175 .clampMaxNumElements(TypeIdx, EltTy, NumElts); 1176 } 1177 1178 /// Fallback on the previous implementation. This should only be used while 1179 /// porting a rule. 1180 LegalizeRuleSet &fallback() { 1181 add({always, LegalizeAction::UseLegacyRules}); 1182 return *this; 1183 } 1184 1185 /// Check if there is no type index which is obviously not handled by the 1186 /// LegalizeRuleSet in any way at all. 1187 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1188 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; 1189 /// Check if there is no imm index which is obviously not handled by the 1190 /// LegalizeRuleSet in any way at all. 1191 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1192 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; 1193 1194 /// Apply the ruleset to the given LegalityQuery. 1195 LegalizeActionStep apply(const LegalityQuery &Query) const; 1196 }; 1197 1198 class LegalizerInfo { 1199 public: 1200 virtual ~LegalizerInfo() = default; 1201 1202 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { 1203 return LegacyInfo; 1204 } 1205 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } 1206 1207 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; 1208 unsigned getActionDefinitionsIdx(unsigned Opcode) const; 1209 1210 /// Perform simple self-diagnostic and assert if there is anything obviously 1211 /// wrong with the actions set up. 1212 void verify(const MCInstrInfo &MII) const; 1213 1214 /// Get the action definitions for the given opcode. Use this to run a 1215 /// LegalityQuery through the definitions. 1216 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; 1217 1218 /// Get the action definition builder for the given opcode. Use this to define 1219 /// the action definitions. 1220 /// 1221 /// It is an error to request an opcode that has already been requested by the 1222 /// multiple-opcode variant. 1223 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); 1224 1225 /// Get the action definition builder for the given set of opcodes. Use this 1226 /// to define the action definitions for multiple opcodes at once. The first 1227 /// opcode given will be considered the representative opcode and will hold 1228 /// the definitions whereas the other opcodes will be configured to refer to 1229 /// the representative opcode. This lowers memory requirements and very 1230 /// slightly improves performance. 1231 /// 1232 /// It would be very easy to introduce unexpected side-effects as a result of 1233 /// this aliasing if it were permitted to request different but intersecting 1234 /// sets of opcodes but that is difficult to keep track of. It is therefore an 1235 /// error to request the same opcode twice using this API, to request an 1236 /// opcode that already has definitions, or to use the single-opcode API on an 1237 /// opcode that has already been requested by this API. 1238 LegalizeRuleSet & 1239 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); 1240 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); 1241 1242 /// Determine what action should be taken to legalize the described 1243 /// instruction. Requires computeTables to have been called. 1244 /// 1245 /// \returns a description of the next legalization step to perform. 1246 LegalizeActionStep getAction(const LegalityQuery &Query) const; 1247 1248 /// Determine what action should be taken to legalize the given generic 1249 /// instruction. 1250 /// 1251 /// \returns a description of the next legalization step to perform. 1252 LegalizeActionStep getAction(const MachineInstr &MI, 1253 const MachineRegisterInfo &MRI) const; 1254 1255 bool isLegal(const LegalityQuery &Query) const { 1256 return getAction(Query).Action == LegalizeAction::Legal; 1257 } 1258 1259 bool isLegalOrCustom(const LegalityQuery &Query) const { 1260 auto Action = getAction(Query).Action; 1261 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom; 1262 } 1263 1264 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; 1265 bool isLegalOrCustom(const MachineInstr &MI, 1266 const MachineRegisterInfo &MRI) const; 1267 1268 /// Called for instructions with the Custom LegalizationAction. 1269 virtual bool legalizeCustom(LegalizerHelper &Helper, 1270 MachineInstr &MI) const { 1271 llvm_unreachable("must implement this if custom action is used"); 1272 } 1273 1274 /// \returns true if MI is either legal or has been legalized and false if not 1275 /// legal. 1276 /// Return true if MI is either legal or has been legalized and false 1277 /// if not legal. 1278 virtual bool legalizeIntrinsic(LegalizerHelper &Helper, 1279 MachineInstr &MI) const { 1280 return true; 1281 } 1282 1283 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while 1284 /// widening a constant of type SmallTy which targets can override. 1285 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which 1286 /// will be the default. 1287 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; 1288 1289 private: 1290 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; 1291 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; 1292 1293 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; 1294 LegacyLegalizerInfo LegacyInfo; 1295 }; 1296 1297 #ifndef NDEBUG 1298 /// Checks that MIR is fully legal, returns an illegal instruction if it's not, 1299 /// nullptr otherwise 1300 const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); 1301 #endif 1302 1303 } // end namespace llvm. 1304 1305 #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 1306