1 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// Interface for Targets to specify which operations they can successfully 10 /// select and how the others should be expanded most efficiently. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 15 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 16 17 #include "llvm/ADT/SmallBitVector.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" 20 #include "llvm/CodeGen/LowLevelType.h" 21 #include "llvm/CodeGen/MachineMemOperand.h" 22 #include "llvm/CodeGen/TargetOpcodes.h" 23 #include "llvm/MC/MCInstrDesc.h" 24 #include "llvm/Support/AtomicOrdering.h" 25 #include "llvm/Support/CommandLine.h" 26 #include <cassert> 27 #include <cstdint> 28 #include <tuple> 29 #include <utility> 30 31 namespace llvm { 32 33 extern cl::opt<bool> DisableGISelLegalityCheck; 34 35 class MachineFunction; 36 class raw_ostream; 37 class LegalizerHelper; 38 class MachineInstr; 39 class MachineRegisterInfo; 40 class MCInstrInfo; 41 42 namespace LegalizeActions { 43 enum LegalizeAction : std::uint8_t { 44 /// The operation is expected to be selectable directly by the target, and 45 /// no transformation is necessary. 46 Legal, 47 48 /// The operation should be synthesized from multiple instructions acting on 49 /// a narrower scalar base-type. For example a 64-bit add might be 50 /// implemented in terms of 32-bit add-with-carry. 51 NarrowScalar, 52 53 /// The operation should be implemented in terms of a wider scalar 54 /// base-type. For example a <2 x s8> add could be implemented as a <2 55 /// x s32> add (ignoring the high bits). 56 WidenScalar, 57 58 /// The (vector) operation should be implemented by splitting it into 59 /// sub-vectors where the operation is legal. For example a <8 x s64> add 60 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover 61 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add 62 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover 63 /// types can be avoided by doing MoreElements first. 64 FewerElements, 65 66 /// The (vector) operation should be implemented by widening the input 67 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is 68 /// rarely legal, but you might perform an <8 x i8> and then only look at 69 /// the first two results. 70 MoreElements, 71 72 /// Perform the operation on a different, but equivalently sized type. 73 Bitcast, 74 75 /// The operation itself must be expressed in terms of simpler actions on 76 /// this target. E.g. a SREM replaced by an SDIV and subtraction. 77 Lower, 78 79 /// The operation should be implemented as a call to some kind of runtime 80 /// support library. For example this usually happens on machines that don't 81 /// support floating-point operations natively. 82 Libcall, 83 84 /// The target wants to do something special with this combination of 85 /// operand and type. A callback will be issued when it is needed. 86 Custom, 87 88 /// This operation is completely unsupported on the target. A programming 89 /// error has occurred. 90 Unsupported, 91 92 /// Sentinel value for when no action was found in the specified table. 93 NotFound, 94 95 /// Fall back onto the old rules. 96 /// TODO: Remove this once we've migrated 97 UseLegacyRules, 98 }; 99 } // end namespace LegalizeActions 100 raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); 101 102 using LegalizeActions::LegalizeAction; 103 104 /// The LegalityQuery object bundles together all the information that's needed 105 /// to decide whether a given operation is legal or not. 106 /// For efficiency, it doesn't make a copy of Types so care must be taken not 107 /// to free it before using the query. 108 struct LegalityQuery { 109 unsigned Opcode; 110 ArrayRef<LLT> Types; 111 112 struct MemDesc { 113 LLT MemoryTy; 114 uint64_t AlignInBits; 115 AtomicOrdering Ordering; 116 117 MemDesc() = default; 118 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering) 119 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {} 120 MemDesc(const MachineMemOperand &MMO) 121 : MemoryTy(MMO.getMemoryType()), 122 AlignInBits(MMO.getAlign().value() * 8), 123 Ordering(MMO.getSuccessOrdering()) {} 124 }; 125 126 /// Operations which require memory can use this to place requirements on the 127 /// memory type for each MMO. 128 ArrayRef<MemDesc> MMODescrs; 129 130 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, 131 const ArrayRef<MemDesc> MMODescrs) 132 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} 133 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) 134 : LegalityQuery(Opcode, Types, {}) {} 135 136 raw_ostream &print(raw_ostream &OS) const; 137 }; 138 139 /// The result of a query. It either indicates a final answer of Legal or 140 /// Unsupported or describes an action that must be taken to make an operation 141 /// more legal. 142 struct LegalizeActionStep { 143 /// The action to take or the final answer. 144 LegalizeAction Action; 145 /// If describing an action, the type index to change. Otherwise zero. 146 unsigned TypeIdx; 147 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. 148 LLT NewType; 149 150 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, 151 const LLT NewType) 152 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} 153 154 LegalizeActionStep(LegacyLegalizeActionStep Step) 155 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { 156 switch (Step.Action) { 157 case LegacyLegalizeActions::Legal: 158 Action = LegalizeActions::Legal; 159 break; 160 case LegacyLegalizeActions::NarrowScalar: 161 Action = LegalizeActions::NarrowScalar; 162 break; 163 case LegacyLegalizeActions::WidenScalar: 164 Action = LegalizeActions::WidenScalar; 165 break; 166 case LegacyLegalizeActions::FewerElements: 167 Action = LegalizeActions::FewerElements; 168 break; 169 case LegacyLegalizeActions::MoreElements: 170 Action = LegalizeActions::MoreElements; 171 break; 172 case LegacyLegalizeActions::Bitcast: 173 Action = LegalizeActions::Bitcast; 174 break; 175 case LegacyLegalizeActions::Lower: 176 Action = LegalizeActions::Lower; 177 break; 178 case LegacyLegalizeActions::Libcall: 179 Action = LegalizeActions::Libcall; 180 break; 181 case LegacyLegalizeActions::Custom: 182 Action = LegalizeActions::Custom; 183 break; 184 case LegacyLegalizeActions::Unsupported: 185 Action = LegalizeActions::Unsupported; 186 break; 187 case LegacyLegalizeActions::NotFound: 188 Action = LegalizeActions::NotFound; 189 break; 190 } 191 } 192 193 bool operator==(const LegalizeActionStep &RHS) const { 194 return std::tie(Action, TypeIdx, NewType) == 195 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); 196 } 197 }; 198 199 using LegalityPredicate = std::function<bool (const LegalityQuery &)>; 200 using LegalizeMutation = 201 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; 202 203 namespace LegalityPredicates { 204 struct TypePairAndMemDesc { 205 LLT Type0; 206 LLT Type1; 207 LLT MemTy; 208 uint64_t Align; 209 210 bool operator==(const TypePairAndMemDesc &Other) const { 211 return Type0 == Other.Type0 && Type1 == Other.Type1 && 212 Align == Other.Align && MemTy == Other.MemTy; 213 } 214 215 /// \returns true if this memory access is legal with for the access described 216 /// by \p Other (The alignment is sufficient for the size and result type). 217 bool isCompatible(const TypePairAndMemDesc &Other) const { 218 return Type0 == Other.Type0 && Type1 == Other.Type1 && 219 Align >= Other.Align && 220 // FIXME: This perhaps should be stricter, but the current legality 221 // rules are written only considering the size. 222 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); 223 } 224 }; 225 226 /// True iff P0 and P1 are true. 227 template<typename Predicate> 228 Predicate all(Predicate P0, Predicate P1) { 229 return [=](const LegalityQuery &Query) { 230 return P0(Query) && P1(Query); 231 }; 232 } 233 /// True iff all given predicates are true. 234 template<typename Predicate, typename... Args> 235 Predicate all(Predicate P0, Predicate P1, Args... args) { 236 return all(all(P0, P1), args...); 237 } 238 239 /// True iff P0 or P1 are true. 240 template<typename Predicate> 241 Predicate any(Predicate P0, Predicate P1) { 242 return [=](const LegalityQuery &Query) { 243 return P0(Query) || P1(Query); 244 }; 245 } 246 /// True iff any given predicates are true. 247 template<typename Predicate, typename... Args> 248 Predicate any(Predicate P0, Predicate P1, Args... args) { 249 return any(any(P0, P1), args...); 250 } 251 252 /// True iff the given type index is the specified type. 253 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); 254 /// True iff the given type index is one of the specified types. 255 LegalityPredicate typeInSet(unsigned TypeIdx, 256 std::initializer_list<LLT> TypesInit); 257 258 /// True iff the given type index is not the specified type. 259 inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { 260 return [=](const LegalityQuery &Query) { 261 return Query.Types[TypeIdx] != Type; 262 }; 263 } 264 265 /// True iff the given types for the given pair of type indexes is one of the 266 /// specified type pairs. 267 LegalityPredicate 268 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, 269 std::initializer_list<std::pair<LLT, LLT>> TypesInit); 270 /// True iff the given types for the given pair of type indexes is one of the 271 /// specified type pairs. 272 LegalityPredicate typePairAndMemDescInSet( 273 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, 274 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); 275 /// True iff the specified type index is a scalar. 276 LegalityPredicate isScalar(unsigned TypeIdx); 277 /// True iff the specified type index is a vector. 278 LegalityPredicate isVector(unsigned TypeIdx); 279 /// True iff the specified type index is a pointer (with any address space). 280 LegalityPredicate isPointer(unsigned TypeIdx); 281 /// True iff the specified type index is a pointer with the specified address 282 /// space. 283 LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); 284 285 /// True if the type index is a vector with element type \p EltTy 286 LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy); 287 288 /// True iff the specified type index is a scalar that's narrower than the given 289 /// size. 290 LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size); 291 292 /// True iff the specified type index is a scalar that's wider than the given 293 /// size. 294 LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size); 295 296 /// True iff the specified type index is a scalar or vector with an element type 297 /// that's narrower than the given size. 298 LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); 299 300 /// True iff the specified type index is a scalar or a vector with an element 301 /// type that's wider than the given size. 302 LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); 303 304 /// True iff the specified type index is a scalar whose size is not a multiple 305 /// of Size. 306 LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size); 307 308 /// True iff the specified type index is a scalar whose size is not a power of 309 /// 2. 310 LegalityPredicate sizeNotPow2(unsigned TypeIdx); 311 312 /// True iff the specified type index is a scalar or vector whose element size 313 /// is not a power of 2. 314 LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); 315 316 /// True if the total bitwidth of the specified type index is \p Size bits. 317 LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size); 318 319 /// True iff the specified type indices are both the same bit size. 320 LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); 321 322 /// True iff the first type index has a larger total bit size than second type 323 /// index. 324 LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); 325 326 /// True iff the first type index has a smaller total bit size than second type 327 /// index. 328 LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); 329 330 /// True iff the specified MMO index has a size (rounded to bytes) that is not a 331 /// power of 2. 332 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); 333 334 /// True iff the specified MMO index has a size that is not an even byte size, 335 /// or that even byte size is not a power of 2. 336 LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx); 337 338 /// True iff the specified type index is a vector whose element count is not a 339 /// power of 2. 340 LegalityPredicate numElementsNotPow2(unsigned TypeIdx); 341 /// True iff the specified MMO index has at an atomic ordering of at Ordering or 342 /// stronger. 343 LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, 344 AtomicOrdering Ordering); 345 } // end namespace LegalityPredicates 346 347 namespace LegalizeMutations { 348 /// Select this specific type for the given type index. 349 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); 350 351 /// Keep the same type as the given type index. 352 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); 353 354 /// Keep the same scalar or element type as the given type index. 355 LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); 356 357 /// Keep the same scalar or element type as the given type. 358 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); 359 360 /// Keep the same scalar or element type as \p TypeIdx, but take the number of 361 /// elements from \p FromTypeIdx. 362 LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx); 363 364 /// Keep the same scalar or element type as \p TypeIdx, but take the number of 365 /// elements from \p Ty. 366 LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty); 367 368 /// Change the scalar size or element size to have the same scalar size as type 369 /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and 370 /// only changes the size. 371 LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx); 372 373 /// Widen the scalar type or vector element type for the given type index to the 374 /// next power of 2. 375 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); 376 377 /// Widen the scalar type or vector element type for the given type index to 378 /// next multiple of \p Size. 379 LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx, 380 unsigned Size); 381 382 /// Add more elements to the type for the given type index to the next power of 383 /// 2. 384 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); 385 /// Break up the vector type for the given type index into the element type. 386 LegalizeMutation scalarize(unsigned TypeIdx); 387 } // end namespace LegalizeMutations 388 389 /// A single rule in a legalizer info ruleset. 390 /// The specified action is chosen when the predicate is true. Where appropriate 391 /// for the action (e.g. for WidenScalar) the new type is selected using the 392 /// given mutator. 393 class LegalizeRule { 394 LegalityPredicate Predicate; 395 LegalizeAction Action; 396 LegalizeMutation Mutation; 397 398 public: 399 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, 400 LegalizeMutation Mutation = nullptr) 401 : Predicate(Predicate), Action(Action), Mutation(Mutation) {} 402 403 /// Test whether the LegalityQuery matches. 404 bool match(const LegalityQuery &Query) const { 405 return Predicate(Query); 406 } 407 408 LegalizeAction getAction() const { return Action; } 409 410 /// Determine the change to make. 411 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { 412 if (Mutation) 413 return Mutation(Query); 414 return std::make_pair(0, LLT{}); 415 } 416 }; 417 418 class LegalizeRuleSet { 419 /// When non-zero, the opcode we are an alias of 420 unsigned AliasOf = 0; 421 /// If true, there is another opcode that aliases this one 422 bool IsAliasedByAnother = false; 423 SmallVector<LegalizeRule, 2> Rules; 424 425 #ifndef NDEBUG 426 /// If bit I is set, this rule set contains a rule that may handle (predicate 427 /// or perform an action upon (or both)) the type index I. The uncertainty 428 /// comes from free-form rules executing user-provided lambda functions. We 429 /// conservatively assume such rules do the right thing and cover all type 430 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs 431 /// to be to distinguish such cases from the cases where all type indices are 432 /// individually handled. 433 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - 434 MCOI::OPERAND_FIRST_GENERIC + 2}; 435 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - 436 MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; 437 #endif 438 439 unsigned typeIdx(unsigned TypeIdx) { 440 assert(TypeIdx <= 441 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && 442 "Type Index is out of bounds"); 443 #ifndef NDEBUG 444 TypeIdxsCovered.set(TypeIdx); 445 #endif 446 return TypeIdx; 447 } 448 449 void markAllIdxsAsCovered() { 450 #ifndef NDEBUG 451 TypeIdxsCovered.set(); 452 ImmIdxsCovered.set(); 453 #endif 454 } 455 456 void add(const LegalizeRule &Rule) { 457 assert(AliasOf == 0 && 458 "RuleSet is aliased, change the representative opcode instead"); 459 Rules.push_back(Rule); 460 } 461 462 static bool always(const LegalityQuery &) { return true; } 463 464 /// Use the given action when the predicate is true. 465 /// Action should not be an action that requires mutation. 466 LegalizeRuleSet &actionIf(LegalizeAction Action, 467 LegalityPredicate Predicate) { 468 add({Predicate, Action}); 469 return *this; 470 } 471 /// Use the given action when the predicate is true. 472 /// Action should be an action that requires mutation. 473 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, 474 LegalizeMutation Mutation) { 475 add({Predicate, Action, Mutation}); 476 return *this; 477 } 478 /// Use the given action when type index 0 is any type in the given list. 479 /// Action should not be an action that requires mutation. 480 LegalizeRuleSet &actionFor(LegalizeAction Action, 481 std::initializer_list<LLT> Types) { 482 using namespace LegalityPredicates; 483 return actionIf(Action, typeInSet(typeIdx(0), Types)); 484 } 485 /// Use the given action when type index 0 is any type in the given list. 486 /// Action should be an action that requires mutation. 487 LegalizeRuleSet &actionFor(LegalizeAction Action, 488 std::initializer_list<LLT> Types, 489 LegalizeMutation Mutation) { 490 using namespace LegalityPredicates; 491 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation); 492 } 493 /// Use the given action when type indexes 0 and 1 is any type pair in the 494 /// given list. 495 /// Action should not be an action that requires mutation. 496 LegalizeRuleSet &actionFor(LegalizeAction Action, 497 std::initializer_list<std::pair<LLT, LLT>> Types) { 498 using namespace LegalityPredicates; 499 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 500 } 501 /// Use the given action when type indexes 0 and 1 is any type pair in the 502 /// given list. 503 /// Action should be an action that requires mutation. 504 LegalizeRuleSet &actionFor(LegalizeAction Action, 505 std::initializer_list<std::pair<LLT, LLT>> Types, 506 LegalizeMutation Mutation) { 507 using namespace LegalityPredicates; 508 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), 509 Mutation); 510 } 511 /// Use the given action when type index 0 is any type in the given list and 512 /// imm index 0 is anything. Action should not be an action that requires 513 /// mutation. 514 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, 515 std::initializer_list<LLT> Types) { 516 using namespace LegalityPredicates; 517 immIdx(0); // Inform verifier imm idx 0 is handled. 518 return actionIf(Action, typeInSet(typeIdx(0), Types)); 519 } 520 521 LegalizeRuleSet &actionForTypeWithAnyImm( 522 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) { 523 using namespace LegalityPredicates; 524 immIdx(0); // Inform verifier imm idx 0 is handled. 525 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 526 } 527 528 /// Use the given action when type indexes 0 and 1 are both in the given list. 529 /// That is, the type pair is in the cartesian product of the list. 530 /// Action should not be an action that requires mutation. 531 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, 532 std::initializer_list<LLT> Types) { 533 using namespace LegalityPredicates; 534 return actionIf(Action, all(typeInSet(typeIdx(0), Types), 535 typeInSet(typeIdx(1), Types))); 536 } 537 /// Use the given action when type indexes 0 and 1 are both in their 538 /// respective lists. 539 /// That is, the type pair is in the cartesian product of the lists 540 /// Action should not be an action that requires mutation. 541 LegalizeRuleSet & 542 actionForCartesianProduct(LegalizeAction Action, 543 std::initializer_list<LLT> Types0, 544 std::initializer_list<LLT> Types1) { 545 using namespace LegalityPredicates; 546 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 547 typeInSet(typeIdx(1), Types1))); 548 } 549 /// Use the given action when type indexes 0, 1, and 2 are all in their 550 /// respective lists. 551 /// That is, the type triple is in the cartesian product of the lists 552 /// Action should not be an action that requires mutation. 553 LegalizeRuleSet &actionForCartesianProduct( 554 LegalizeAction Action, std::initializer_list<LLT> Types0, 555 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { 556 using namespace LegalityPredicates; 557 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 558 all(typeInSet(typeIdx(1), Types1), 559 typeInSet(typeIdx(2), Types2)))); 560 } 561 562 public: 563 LegalizeRuleSet() = default; 564 565 bool isAliasedByAnother() { return IsAliasedByAnother; } 566 void setIsAliasedByAnother() { IsAliasedByAnother = true; } 567 void aliasTo(unsigned Opcode) { 568 assert((AliasOf == 0 || AliasOf == Opcode) && 569 "Opcode is already aliased to another opcode"); 570 assert(Rules.empty() && "Aliasing will discard rules"); 571 AliasOf = Opcode; 572 } 573 unsigned getAlias() const { return AliasOf; } 574 575 unsigned immIdx(unsigned ImmIdx) { 576 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - 577 MCOI::OPERAND_FIRST_GENERIC_IMM) && 578 "Imm Index is out of bounds"); 579 #ifndef NDEBUG 580 ImmIdxsCovered.set(ImmIdx); 581 #endif 582 return ImmIdx; 583 } 584 585 /// The instruction is legal if predicate is true. 586 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { 587 // We have no choice but conservatively assume that the free-form 588 // user-provided Predicate properly handles all type indices: 589 markAllIdxsAsCovered(); 590 return actionIf(LegalizeAction::Legal, Predicate); 591 } 592 /// The instruction is legal when type index 0 is any type in the given list. 593 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { 594 return actionFor(LegalizeAction::Legal, Types); 595 } 596 /// The instruction is legal when type indexes 0 and 1 is any type pair in the 597 /// given list. 598 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 599 return actionFor(LegalizeAction::Legal, Types); 600 } 601 /// The instruction is legal when type index 0 is any type in the given list 602 /// and imm index 0 is anything. 603 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { 604 markAllIdxsAsCovered(); 605 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 606 } 607 608 LegalizeRuleSet &legalForTypeWithAnyImm( 609 std::initializer_list<std::pair<LLT, LLT>> Types) { 610 markAllIdxsAsCovered(); 611 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 612 } 613 614 /// The instruction is legal when type indexes 0 and 1 along with the memory 615 /// size and minimum alignment is any type and size tuple in the given list. 616 LegalizeRuleSet &legalForTypesWithMemDesc( 617 std::initializer_list<LegalityPredicates::TypePairAndMemDesc> 618 TypesAndMemDesc) { 619 return actionIf(LegalizeAction::Legal, 620 LegalityPredicates::typePairAndMemDescInSet( 621 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); 622 } 623 /// The instruction is legal when type indexes 0 and 1 are both in the given 624 /// list. That is, the type pair is in the cartesian product of the list. 625 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { 626 return actionForCartesianProduct(LegalizeAction::Legal, Types); 627 } 628 /// The instruction is legal when type indexes 0 and 1 are both their 629 /// respective lists. 630 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 631 std::initializer_list<LLT> Types1) { 632 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); 633 } 634 /// The instruction is legal when type indexes 0, 1, and 2 are both their 635 /// respective lists. 636 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 637 std::initializer_list<LLT> Types1, 638 std::initializer_list<LLT> Types2) { 639 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, 640 Types2); 641 } 642 643 LegalizeRuleSet &alwaysLegal() { 644 using namespace LegalizeMutations; 645 markAllIdxsAsCovered(); 646 return actionIf(LegalizeAction::Legal, always); 647 } 648 649 /// The specified type index is coerced if predicate is true. 650 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, 651 LegalizeMutation Mutation) { 652 // We have no choice but conservatively assume that lowering with a 653 // free-form user provided Predicate properly handles all type indices: 654 markAllIdxsAsCovered(); 655 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation); 656 } 657 658 /// The instruction is lowered. 659 LegalizeRuleSet &lower() { 660 using namespace LegalizeMutations; 661 // We have no choice but conservatively assume that predicate-less lowering 662 // properly handles all type indices by design: 663 markAllIdxsAsCovered(); 664 return actionIf(LegalizeAction::Lower, always); 665 } 666 /// The instruction is lowered if predicate is true. Keep type index 0 as the 667 /// same type. 668 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { 669 using namespace LegalizeMutations; 670 // We have no choice but conservatively assume that lowering with a 671 // free-form user provided Predicate properly handles all type indices: 672 markAllIdxsAsCovered(); 673 return actionIf(LegalizeAction::Lower, Predicate); 674 } 675 /// The instruction is lowered if predicate is true. 676 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, 677 LegalizeMutation Mutation) { 678 // We have no choice but conservatively assume that lowering with a 679 // free-form user provided Predicate properly handles all type indices: 680 markAllIdxsAsCovered(); 681 return actionIf(LegalizeAction::Lower, Predicate, Mutation); 682 } 683 /// The instruction is lowered when type index 0 is any type in the given 684 /// list. Keep type index 0 as the same type. 685 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { 686 return actionFor(LegalizeAction::Lower, Types); 687 } 688 /// The instruction is lowered when type index 0 is any type in the given 689 /// list. 690 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, 691 LegalizeMutation Mutation) { 692 return actionFor(LegalizeAction::Lower, Types, Mutation); 693 } 694 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 695 /// the given list. Keep type index 0 as the same type. 696 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 697 return actionFor(LegalizeAction::Lower, Types); 698 } 699 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 700 /// the given list. 701 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, 702 LegalizeMutation Mutation) { 703 return actionFor(LegalizeAction::Lower, Types, Mutation); 704 } 705 /// The instruction is lowered when type indexes 0 and 1 are both in their 706 /// respective lists. 707 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 708 std::initializer_list<LLT> Types1) { 709 using namespace LegalityPredicates; 710 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1); 711 } 712 /// The instruction is lowered when when type indexes 0, 1, and 2 are all in 713 /// their respective lists. 714 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 715 std::initializer_list<LLT> Types1, 716 std::initializer_list<LLT> Types2) { 717 using namespace LegalityPredicates; 718 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1, 719 Types2); 720 } 721 722 /// The instruction is emitted as a library call. 723 LegalizeRuleSet &libcall() { 724 using namespace LegalizeMutations; 725 // We have no choice but conservatively assume that predicate-less lowering 726 // properly handles all type indices by design: 727 markAllIdxsAsCovered(); 728 return actionIf(LegalizeAction::Libcall, always); 729 } 730 731 /// Like legalIf, but for the Libcall action. 732 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { 733 // We have no choice but conservatively assume that a libcall with a 734 // free-form user provided Predicate properly handles all type indices: 735 markAllIdxsAsCovered(); 736 return actionIf(LegalizeAction::Libcall, Predicate); 737 } 738 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { 739 return actionFor(LegalizeAction::Libcall, Types); 740 } 741 LegalizeRuleSet & 742 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 743 return actionFor(LegalizeAction::Libcall, Types); 744 } 745 LegalizeRuleSet & 746 libcallForCartesianProduct(std::initializer_list<LLT> Types) { 747 return actionForCartesianProduct(LegalizeAction::Libcall, Types); 748 } 749 LegalizeRuleSet & 750 libcallForCartesianProduct(std::initializer_list<LLT> Types0, 751 std::initializer_list<LLT> Types1) { 752 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); 753 } 754 755 /// Widen the scalar to the one selected by the mutation if the predicate is 756 /// true. 757 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, 758 LegalizeMutation Mutation) { 759 // We have no choice but conservatively assume that an action with a 760 // free-form user provided Predicate properly handles all type indices: 761 markAllIdxsAsCovered(); 762 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); 763 } 764 /// Narrow the scalar to the one selected by the mutation if the predicate is 765 /// true. 766 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, 767 LegalizeMutation Mutation) { 768 // We have no choice but conservatively assume that an action with a 769 // free-form user provided Predicate properly handles all type indices: 770 markAllIdxsAsCovered(); 771 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); 772 } 773 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any 774 /// type pair in the given list. 775 LegalizeRuleSet & 776 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types, 777 LegalizeMutation Mutation) { 778 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation); 779 } 780 781 /// Add more elements to reach the type selected by the mutation if the 782 /// predicate is true. 783 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, 784 LegalizeMutation Mutation) { 785 // We have no choice but conservatively assume that an action with a 786 // free-form user provided Predicate properly handles all type indices: 787 markAllIdxsAsCovered(); 788 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); 789 } 790 /// Remove elements to reach the type selected by the mutation if the 791 /// predicate is true. 792 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, 793 LegalizeMutation Mutation) { 794 // We have no choice but conservatively assume that an action with a 795 // free-form user provided Predicate properly handles all type indices: 796 markAllIdxsAsCovered(); 797 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); 798 } 799 800 /// The instruction is unsupported. 801 LegalizeRuleSet &unsupported() { 802 markAllIdxsAsCovered(); 803 return actionIf(LegalizeAction::Unsupported, always); 804 } 805 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { 806 return actionIf(LegalizeAction::Unsupported, Predicate); 807 } 808 809 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) { 810 return actionFor(LegalizeAction::Unsupported, Types); 811 } 812 813 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { 814 return actionIf(LegalizeAction::Unsupported, 815 LegalityPredicates::memSizeInBytesNotPow2(0)); 816 } 817 818 /// Lower a memory operation if the memory size, rounded to bytes, is not a 819 /// power of 2. For example, this will not trigger for s1 or s7, but will for 820 /// s24. 821 LegalizeRuleSet &lowerIfMemSizeNotPow2() { 822 return actionIf(LegalizeAction::Lower, 823 LegalityPredicates::memSizeInBytesNotPow2(0)); 824 } 825 826 /// Lower a memory operation if the memory access size is not a round power of 827 /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely 828 /// what you want (e.g. this will lower s1, s7 and s24). 829 LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() { 830 return actionIf(LegalizeAction::Lower, 831 LegalityPredicates::memSizeNotByteSizePow2(0)); 832 } 833 834 LegalizeRuleSet &customIf(LegalityPredicate Predicate) { 835 // We have no choice but conservatively assume that a custom action with a 836 // free-form user provided Predicate properly handles all type indices: 837 markAllIdxsAsCovered(); 838 return actionIf(LegalizeAction::Custom, Predicate); 839 } 840 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { 841 return actionFor(LegalizeAction::Custom, Types); 842 } 843 844 /// The instruction is custom when type indexes 0 and 1 is any type pair in the 845 /// given list. 846 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 847 return actionFor(LegalizeAction::Custom, Types); 848 } 849 850 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { 851 return actionForCartesianProduct(LegalizeAction::Custom, Types); 852 } 853 /// The instruction is custom when type indexes 0 and 1 are both in their 854 /// respective lists. 855 LegalizeRuleSet & 856 customForCartesianProduct(std::initializer_list<LLT> Types0, 857 std::initializer_list<LLT> Types1) { 858 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); 859 } 860 /// The instruction is custom when when type indexes 0, 1, and 2 are all in 861 /// their respective lists. 862 LegalizeRuleSet & 863 customForCartesianProduct(std::initializer_list<LLT> Types0, 864 std::initializer_list<LLT> Types1, 865 std::initializer_list<LLT> Types2) { 866 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1, 867 Types2); 868 } 869 870 /// Unconditionally custom lower. 871 LegalizeRuleSet &custom() { 872 return customIf(always); 873 } 874 875 /// Widen the scalar to the next power of two that is at least MinSize. 876 /// No effect if the type is not a scalar or is a power of two. 877 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, 878 unsigned MinSize = 0) { 879 using namespace LegalityPredicates; 880 return actionIf( 881 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), 882 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 883 } 884 885 /// Widen the scalar to the next multiple of Size. No effect if the 886 /// type is not a scalar or is a multiple of Size. 887 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx, 888 unsigned Size) { 889 using namespace LegalityPredicates; 890 return actionIf( 891 LegalizeAction::WidenScalar, sizeNotMultipleOf(typeIdx(TypeIdx), Size), 892 LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size)); 893 } 894 895 /// Widen the scalar or vector element type to the next power of two that is 896 /// at least MinSize. No effect if the scalar size is a power of two. 897 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, 898 unsigned MinSize = 0) { 899 using namespace LegalityPredicates; 900 return actionIf( 901 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), 902 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 903 } 904 905 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { 906 using namespace LegalityPredicates; 907 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)), 908 Mutation); 909 } 910 911 LegalizeRuleSet &scalarize(unsigned TypeIdx) { 912 using namespace LegalityPredicates; 913 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), 914 LegalizeMutations::scalarize(TypeIdx)); 915 } 916 917 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) { 918 using namespace LegalityPredicates; 919 return actionIf(LegalizeAction::FewerElements, 920 all(Predicate, isVector(typeIdx(TypeIdx))), 921 LegalizeMutations::scalarize(TypeIdx)); 922 } 923 924 /// Ensure the scalar or element is at least as wide as Ty. 925 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) { 926 using namespace LegalityPredicates; 927 using namespace LegalizeMutations; 928 return actionIf(LegalizeAction::WidenScalar, 929 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), 930 changeElementTo(typeIdx(TypeIdx), Ty)); 931 } 932 933 /// Ensure the scalar or element is at least as wide as Ty. 934 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, 935 unsigned TypeIdx, const LLT Ty) { 936 using namespace LegalityPredicates; 937 using namespace LegalizeMutations; 938 return actionIf(LegalizeAction::WidenScalar, 939 all(Predicate, scalarOrEltNarrowerThan( 940 TypeIdx, Ty.getScalarSizeInBits())), 941 changeElementTo(typeIdx(TypeIdx), Ty)); 942 } 943 944 /// Ensure the vector size is at least as wide as VectorSize by promoting the 945 /// element. 946 LegalizeRuleSet &widenVectorEltsToVectorMinSize(unsigned TypeIdx, 947 unsigned VectorSize) { 948 using namespace LegalityPredicates; 949 using namespace LegalizeMutations; 950 return actionIf( 951 LegalizeAction::WidenScalar, 952 [=](const LegalityQuery &Query) { 953 const LLT VecTy = Query.Types[TypeIdx]; 954 return VecTy.isVector() && !VecTy.isScalable() && 955 VecTy.getSizeInBits() < VectorSize; 956 }, 957 [=](const LegalityQuery &Query) { 958 const LLT VecTy = Query.Types[TypeIdx]; 959 unsigned NumElts = VecTy.getNumElements(); 960 unsigned MinSize = VectorSize / NumElts; 961 LLT NewTy = LLT::fixed_vector(NumElts, LLT::scalar(MinSize)); 962 return std::make_pair(TypeIdx, NewTy); 963 }); 964 } 965 966 /// Ensure the scalar is at least as wide as Ty. 967 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) { 968 using namespace LegalityPredicates; 969 using namespace LegalizeMutations; 970 return actionIf(LegalizeAction::WidenScalar, 971 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()), 972 changeTo(typeIdx(TypeIdx), Ty)); 973 } 974 975 /// Ensure the scalar is at least as wide as Ty if condition is met. 976 LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 977 const LLT Ty) { 978 using namespace LegalityPredicates; 979 using namespace LegalizeMutations; 980 return actionIf( 981 LegalizeAction::WidenScalar, 982 [=](const LegalityQuery &Query) { 983 const LLT QueryTy = Query.Types[TypeIdx]; 984 return QueryTy.isScalar() && 985 QueryTy.getSizeInBits() < Ty.getSizeInBits() && 986 Predicate(Query); 987 }, 988 changeTo(typeIdx(TypeIdx), Ty)); 989 } 990 991 /// Ensure the scalar is at most as wide as Ty. 992 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) { 993 using namespace LegalityPredicates; 994 using namespace LegalizeMutations; 995 return actionIf(LegalizeAction::NarrowScalar, 996 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), 997 changeElementTo(typeIdx(TypeIdx), Ty)); 998 } 999 1000 /// Ensure the scalar is at most as wide as Ty. 1001 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) { 1002 using namespace LegalityPredicates; 1003 using namespace LegalizeMutations; 1004 return actionIf(LegalizeAction::NarrowScalar, 1005 scalarWiderThan(TypeIdx, Ty.getSizeInBits()), 1006 changeTo(typeIdx(TypeIdx), Ty)); 1007 } 1008 1009 /// Conditionally limit the maximum size of the scalar. 1010 /// For example, when the maximum size of one type depends on the size of 1011 /// another such as extracting N bits from an M bit container. 1012 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 1013 const LLT Ty) { 1014 using namespace LegalityPredicates; 1015 using namespace LegalizeMutations; 1016 return actionIf( 1017 LegalizeAction::NarrowScalar, 1018 [=](const LegalityQuery &Query) { 1019 const LLT QueryTy = Query.Types[TypeIdx]; 1020 return QueryTy.isScalar() && 1021 QueryTy.getSizeInBits() > Ty.getSizeInBits() && 1022 Predicate(Query); 1023 }, 1024 changeElementTo(typeIdx(TypeIdx), Ty)); 1025 } 1026 1027 /// Limit the range of scalar sizes to MinTy and MaxTy. 1028 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy, 1029 const LLT MaxTy) { 1030 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); 1031 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); 1032 } 1033 1034 /// Limit the range of scalar sizes to MinTy and MaxTy. 1035 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, 1036 const LLT MaxTy) { 1037 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); 1038 } 1039 1040 /// Widen the scalar to match the size of another. 1041 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { 1042 typeIdx(TypeIdx); 1043 return widenScalarIf( 1044 [=](const LegalityQuery &Query) { 1045 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1046 Query.Types[TypeIdx].getSizeInBits(); 1047 }, 1048 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx)); 1049 } 1050 1051 /// Narrow the scalar to match the size of another. 1052 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) { 1053 typeIdx(TypeIdx); 1054 return narrowScalarIf( 1055 [=](const LegalityQuery &Query) { 1056 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() < 1057 Query.Types[TypeIdx].getSizeInBits(); 1058 }, 1059 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx)); 1060 } 1061 1062 /// Change the type \p TypeIdx to have the same scalar size as type \p 1063 /// SameSizeIdx. 1064 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) { 1065 return minScalarSameAs(TypeIdx, SameSizeIdx) 1066 .maxScalarSameAs(TypeIdx, SameSizeIdx); 1067 } 1068 1069 /// Conditionally widen the scalar or elt to match the size of another. 1070 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, 1071 unsigned TypeIdx, unsigned LargeTypeIdx) { 1072 typeIdx(TypeIdx); 1073 return widenScalarIf( 1074 [=](const LegalityQuery &Query) { 1075 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1076 Query.Types[TypeIdx].getScalarSizeInBits() && 1077 Predicate(Query); 1078 }, 1079 [=](const LegalityQuery &Query) { 1080 LLT T = Query.Types[LargeTypeIdx]; 1081 if (T.isVector() && T.getElementType().isPointer()) 1082 T = T.changeElementType(LLT::scalar(T.getScalarSizeInBits())); 1083 return std::make_pair(TypeIdx, T); 1084 }); 1085 } 1086 1087 /// Conditionally narrow the scalar or elt to match the size of another. 1088 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, 1089 unsigned TypeIdx, 1090 unsigned SmallTypeIdx) { 1091 typeIdx(TypeIdx); 1092 return narrowScalarIf( 1093 [=](const LegalityQuery &Query) { 1094 return Query.Types[SmallTypeIdx].getScalarSizeInBits() < 1095 Query.Types[TypeIdx].getScalarSizeInBits() && 1096 Predicate(Query); 1097 }, 1098 [=](const LegalityQuery &Query) { 1099 LLT T = Query.Types[SmallTypeIdx]; 1100 return std::make_pair(TypeIdx, T); 1101 }); 1102 } 1103 1104 /// Add more elements to the vector to reach the next power of two. 1105 /// No effect if the type is not a vector or the element count is a power of 1106 /// two. 1107 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { 1108 using namespace LegalityPredicates; 1109 return actionIf(LegalizeAction::MoreElements, 1110 numElementsNotPow2(typeIdx(TypeIdx)), 1111 LegalizeMutations::moreElementsToNextPow2(TypeIdx)); 1112 } 1113 1114 /// Limit the number of elements in EltTy vectors to at least MinElements. 1115 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy, 1116 unsigned MinElements) { 1117 // Mark the type index as covered: 1118 typeIdx(TypeIdx); 1119 return actionIf( 1120 LegalizeAction::MoreElements, 1121 [=](const LegalityQuery &Query) { 1122 LLT VecTy = Query.Types[TypeIdx]; 1123 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1124 VecTy.getNumElements() < MinElements; 1125 }, 1126 [=](const LegalityQuery &Query) { 1127 LLT VecTy = Query.Types[TypeIdx]; 1128 return std::make_pair( 1129 TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType())); 1130 }); 1131 } 1132 1133 /// Set number of elements to nearest larger multiple of NumElts. 1134 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy, 1135 unsigned NumElts) { 1136 typeIdx(TypeIdx); 1137 return actionIf( 1138 LegalizeAction::MoreElements, 1139 [=](const LegalityQuery &Query) { 1140 LLT VecTy = Query.Types[TypeIdx]; 1141 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1142 (VecTy.getNumElements() % NumElts != 0); 1143 }, 1144 [=](const LegalityQuery &Query) { 1145 LLT VecTy = Query.Types[TypeIdx]; 1146 unsigned NewSize = alignTo(VecTy.getNumElements(), NumElts); 1147 return std::make_pair( 1148 TypeIdx, LLT::fixed_vector(NewSize, VecTy.getElementType())); 1149 }); 1150 } 1151 1152 /// Limit the number of elements in EltTy vectors to at most MaxElements. 1153 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, 1154 unsigned MaxElements) { 1155 // Mark the type index as covered: 1156 typeIdx(TypeIdx); 1157 return actionIf( 1158 LegalizeAction::FewerElements, 1159 [=](const LegalityQuery &Query) { 1160 LLT VecTy = Query.Types[TypeIdx]; 1161 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1162 VecTy.getNumElements() > MaxElements; 1163 }, 1164 [=](const LegalityQuery &Query) { 1165 LLT VecTy = Query.Types[TypeIdx]; 1166 LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements), 1167 VecTy.getElementType()); 1168 return std::make_pair(TypeIdx, NewTy); 1169 }); 1170 } 1171 /// Limit the number of elements for the given vectors to at least MinTy's 1172 /// number of elements and at most MaxTy's number of elements. 1173 /// 1174 /// No effect if the type is not a vector or does not have the same element 1175 /// type as the constraints. 1176 /// The element type of MinTy and MaxTy must match. 1177 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy, 1178 const LLT MaxTy) { 1179 assert(MinTy.getElementType() == MaxTy.getElementType() && 1180 "Expected element types to agree"); 1181 1182 const LLT EltTy = MinTy.getElementType(); 1183 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) 1184 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); 1185 } 1186 1187 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements 1188 /// (or scalars when \p NumElts equals 1). 1189 /// First pad with undef elements to nearest larger multiple of \p NumElts. 1190 /// Then perform split with all sub-instructions having the same type. 1191 /// Using clampMaxNumElements (non-strict) can result in leftover instruction 1192 /// with different type (fewer elements then \p NumElts or scalar). 1193 /// No effect if the type is not a vector. 1194 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, 1195 unsigned NumElts) { 1196 return alignNumElementsTo(TypeIdx, EltTy, NumElts) 1197 .clampMaxNumElements(TypeIdx, EltTy, NumElts); 1198 } 1199 1200 /// Fallback on the previous implementation. This should only be used while 1201 /// porting a rule. 1202 LegalizeRuleSet &fallback() { 1203 add({always, LegalizeAction::UseLegacyRules}); 1204 return *this; 1205 } 1206 1207 /// Check if there is no type index which is obviously not handled by the 1208 /// LegalizeRuleSet in any way at all. 1209 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1210 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; 1211 /// Check if there is no imm index which is obviously not handled by the 1212 /// LegalizeRuleSet in any way at all. 1213 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1214 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; 1215 1216 /// Apply the ruleset to the given LegalityQuery. 1217 LegalizeActionStep apply(const LegalityQuery &Query) const; 1218 }; 1219 1220 class LegalizerInfo { 1221 public: 1222 virtual ~LegalizerInfo() = default; 1223 1224 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { 1225 return LegacyInfo; 1226 } 1227 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } 1228 1229 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; 1230 unsigned getActionDefinitionsIdx(unsigned Opcode) const; 1231 1232 /// Perform simple self-diagnostic and assert if there is anything obviously 1233 /// wrong with the actions set up. 1234 void verify(const MCInstrInfo &MII) const; 1235 1236 /// Get the action definitions for the given opcode. Use this to run a 1237 /// LegalityQuery through the definitions. 1238 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; 1239 1240 /// Get the action definition builder for the given opcode. Use this to define 1241 /// the action definitions. 1242 /// 1243 /// It is an error to request an opcode that has already been requested by the 1244 /// multiple-opcode variant. 1245 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); 1246 1247 /// Get the action definition builder for the given set of opcodes. Use this 1248 /// to define the action definitions for multiple opcodes at once. The first 1249 /// opcode given will be considered the representative opcode and will hold 1250 /// the definitions whereas the other opcodes will be configured to refer to 1251 /// the representative opcode. This lowers memory requirements and very 1252 /// slightly improves performance. 1253 /// 1254 /// It would be very easy to introduce unexpected side-effects as a result of 1255 /// this aliasing if it were permitted to request different but intersecting 1256 /// sets of opcodes but that is difficult to keep track of. It is therefore an 1257 /// error to request the same opcode twice using this API, to request an 1258 /// opcode that already has definitions, or to use the single-opcode API on an 1259 /// opcode that has already been requested by this API. 1260 LegalizeRuleSet & 1261 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); 1262 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); 1263 1264 /// Determine what action should be taken to legalize the described 1265 /// instruction. Requires computeTables to have been called. 1266 /// 1267 /// \returns a description of the next legalization step to perform. 1268 LegalizeActionStep getAction(const LegalityQuery &Query) const; 1269 1270 /// Determine what action should be taken to legalize the given generic 1271 /// instruction. 1272 /// 1273 /// \returns a description of the next legalization step to perform. 1274 LegalizeActionStep getAction(const MachineInstr &MI, 1275 const MachineRegisterInfo &MRI) const; 1276 1277 bool isLegal(const LegalityQuery &Query) const { 1278 return getAction(Query).Action == LegalizeAction::Legal; 1279 } 1280 1281 bool isLegalOrCustom(const LegalityQuery &Query) const { 1282 auto Action = getAction(Query).Action; 1283 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom; 1284 } 1285 1286 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; 1287 bool isLegalOrCustom(const MachineInstr &MI, 1288 const MachineRegisterInfo &MRI) const; 1289 1290 /// Called for instructions with the Custom LegalizationAction. 1291 virtual bool legalizeCustom(LegalizerHelper &Helper, 1292 MachineInstr &MI) const { 1293 llvm_unreachable("must implement this if custom action is used"); 1294 } 1295 1296 /// \returns true if MI is either legal or has been legalized and false if not 1297 /// legal. 1298 /// Return true if MI is either legal or has been legalized and false 1299 /// if not legal. 1300 virtual bool legalizeIntrinsic(LegalizerHelper &Helper, 1301 MachineInstr &MI) const { 1302 return true; 1303 } 1304 1305 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while 1306 /// widening a constant of type SmallTy which targets can override. 1307 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which 1308 /// will be the default. 1309 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; 1310 1311 private: 1312 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; 1313 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; 1314 1315 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; 1316 LegacyLegalizerInfo LegacyInfo; 1317 }; 1318 1319 #ifndef NDEBUG 1320 /// Checks that MIR is fully legal, returns an illegal instruction if it's not, 1321 /// nullptr otherwise 1322 const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); 1323 #endif 1324 1325 } // end namespace llvm. 1326 1327 #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 1328