1 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// Interface for Targets to specify which operations they can successfully
10 /// select and how the others should be expanded most efficiently.
11 ///
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
15 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
16
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/TargetOpcodes.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/LowLevelTypeImpl.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <tuple>
32 #include <unordered_map>
33 #include <utility>
34
35 namespace llvm {
36
37 extern cl::opt<bool> DisableGISelLegalityCheck;
38
39 class LegalizerHelper;
40 class MachineInstr;
41 class MachineRegisterInfo;
42 class MCInstrInfo;
43 class GISelChangeObserver;
44
45 namespace LegalizeActions {
46 enum LegalizeAction : std::uint8_t {
47 /// The operation is expected to be selectable directly by the target, and
48 /// no transformation is necessary.
49 Legal,
50
51 /// The operation should be synthesized from multiple instructions acting on
52 /// a narrower scalar base-type. For example a 64-bit add might be
53 /// implemented in terms of 32-bit add-with-carry.
54 NarrowScalar,
55
56 /// The operation should be implemented in terms of a wider scalar
57 /// base-type. For example a <2 x s8> add could be implemented as a <2
58 /// x s32> add (ignoring the high bits).
59 WidenScalar,
60
61 /// The (vector) operation should be implemented by splitting it into
62 /// sub-vectors where the operation is legal. For example a <8 x s64> add
63 /// might be implemented as 4 separate <2 x s64> adds.
64 FewerElements,
65
66 /// The (vector) operation should be implemented by widening the input
67 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is
68 /// rarely legal, but you might perform an <8 x i8> and then only look at
69 /// the first two results.
70 MoreElements,
71
72 /// Perform the operation on a different, but equivalently sized type.
73 Bitcast,
74
75 /// The operation itself must be expressed in terms of simpler actions on
76 /// this target. E.g. a SREM replaced by an SDIV and subtraction.
77 Lower,
78
79 /// The operation should be implemented as a call to some kind of runtime
80 /// support library. For example this usually happens on machines that don't
81 /// support floating-point operations natively.
82 Libcall,
83
84 /// The target wants to do something special with this combination of
85 /// operand and type. A callback will be issued when it is needed.
86 Custom,
87
88 /// This operation is completely unsupported on the target. A programming
89 /// error has occurred.
90 Unsupported,
91
92 /// Sentinel value for when no action was found in the specified table.
93 NotFound,
94
95 /// Fall back onto the old rules.
96 /// TODO: Remove this once we've migrated
97 UseLegacyRules,
98 };
99 } // end namespace LegalizeActions
100 raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action);
101
102 using LegalizeActions::LegalizeAction;
103
104 /// The LegalityQuery object bundles together all the information that's needed
105 /// to decide whether a given operation is legal or not.
106 /// For efficiency, it doesn't make a copy of Types so care must be taken not
107 /// to free it before using the query.
108 struct LegalityQuery {
109 unsigned Opcode;
110 ArrayRef<LLT> Types;
111
112 struct MemDesc {
113 LLT MemoryTy;
114 uint64_t AlignInBits;
115 AtomicOrdering Ordering;
116 };
117
118 /// Operations which require memory can use this to place requirements on the
119 /// memory type for each MMO.
120 ArrayRef<MemDesc> MMODescrs;
121
LegalityQueryLegalityQuery122 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
123 const ArrayRef<MemDesc> MMODescrs)
124 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
LegalityQueryLegalityQuery125 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
126 : LegalityQuery(Opcode, Types, {}) {}
127
128 raw_ostream &print(raw_ostream &OS) const;
129 };
130
131 /// The result of a query. It either indicates a final answer of Legal or
132 /// Unsupported or describes an action that must be taken to make an operation
133 /// more legal.
134 struct LegalizeActionStep {
135 /// The action to take or the final answer.
136 LegalizeAction Action;
137 /// If describing an action, the type index to change. Otherwise zero.
138 unsigned TypeIdx;
139 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}.
140 LLT NewType;
141
LegalizeActionStepLegalizeActionStep142 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx,
143 const LLT NewType)
144 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {}
145
LegalizeActionStepLegalizeActionStep146 LegalizeActionStep(LegacyLegalizeActionStep Step)
147 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) {
148 switch (Step.Action) {
149 case LegacyLegalizeActions::Legal:
150 Action = LegalizeActions::Legal;
151 break;
152 case LegacyLegalizeActions::NarrowScalar:
153 Action = LegalizeActions::NarrowScalar;
154 break;
155 case LegacyLegalizeActions::WidenScalar:
156 Action = LegalizeActions::WidenScalar;
157 break;
158 case LegacyLegalizeActions::FewerElements:
159 Action = LegalizeActions::FewerElements;
160 break;
161 case LegacyLegalizeActions::MoreElements:
162 Action = LegalizeActions::MoreElements;
163 break;
164 case LegacyLegalizeActions::Bitcast:
165 Action = LegalizeActions::Bitcast;
166 break;
167 case LegacyLegalizeActions::Lower:
168 Action = LegalizeActions::Lower;
169 break;
170 case LegacyLegalizeActions::Libcall:
171 Action = LegalizeActions::Libcall;
172 break;
173 case LegacyLegalizeActions::Custom:
174 Action = LegalizeActions::Custom;
175 break;
176 case LegacyLegalizeActions::Unsupported:
177 Action = LegalizeActions::Unsupported;
178 break;
179 case LegacyLegalizeActions::NotFound:
180 Action = LegalizeActions::NotFound;
181 break;
182 }
183 }
184
185 bool operator==(const LegalizeActionStep &RHS) const {
186 return std::tie(Action, TypeIdx, NewType) ==
187 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType);
188 }
189 };
190
191 using LegalityPredicate = std::function<bool (const LegalityQuery &)>;
192 using LegalizeMutation =
193 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
194
195 namespace LegalityPredicates {
196 struct TypePairAndMemDesc {
197 LLT Type0;
198 LLT Type1;
199 LLT MemTy;
200 uint64_t Align;
201
202 bool operator==(const TypePairAndMemDesc &Other) const {
203 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
204 Align == Other.Align && MemTy == Other.MemTy;
205 }
206
207 /// \returns true if this memory access is legal with for the access described
208 /// by \p Other (The alignment is sufficient for the size and result type).
isCompatibleTypePairAndMemDesc209 bool isCompatible(const TypePairAndMemDesc &Other) const {
210 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
211 Align >= Other.Align &&
212 // FIXME: This perhaps should be stricter, but the current legality
213 // rules are written only considering the size.
214 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits();
215 }
216 };
217
218 /// True iff P0 and P1 are true.
219 template<typename Predicate>
all(Predicate P0,Predicate P1)220 Predicate all(Predicate P0, Predicate P1) {
221 return [=](const LegalityQuery &Query) {
222 return P0(Query) && P1(Query);
223 };
224 }
225 /// True iff all given predicates are true.
226 template<typename Predicate, typename... Args>
all(Predicate P0,Predicate P1,Args...args)227 Predicate all(Predicate P0, Predicate P1, Args... args) {
228 return all(all(P0, P1), args...);
229 }
230
231 /// True iff P0 or P1 are true.
232 template<typename Predicate>
any(Predicate P0,Predicate P1)233 Predicate any(Predicate P0, Predicate P1) {
234 return [=](const LegalityQuery &Query) {
235 return P0(Query) || P1(Query);
236 };
237 }
238 /// True iff any given predicates are true.
239 template<typename Predicate, typename... Args>
any(Predicate P0,Predicate P1,Args...args)240 Predicate any(Predicate P0, Predicate P1, Args... args) {
241 return any(any(P0, P1), args...);
242 }
243
244 /// True iff the given type index is the specified type.
245 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
246 /// True iff the given type index is one of the specified types.
247 LegalityPredicate typeInSet(unsigned TypeIdx,
248 std::initializer_list<LLT> TypesInit);
249
250 /// True iff the given type index is not the specified type.
typeIsNot(unsigned TypeIdx,LLT Type)251 inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
252 return [=](const LegalityQuery &Query) {
253 return Query.Types[TypeIdx] != Type;
254 };
255 }
256
257 /// True iff the given types for the given pair of type indexes is one of the
258 /// specified type pairs.
259 LegalityPredicate
260 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
261 std::initializer_list<std::pair<LLT, LLT>> TypesInit);
262 /// True iff the given types for the given pair of type indexes is one of the
263 /// specified type pairs.
264 LegalityPredicate typePairAndMemDescInSet(
265 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
266 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit);
267 /// True iff the specified type index is a scalar.
268 LegalityPredicate isScalar(unsigned TypeIdx);
269 /// True iff the specified type index is a vector.
270 LegalityPredicate isVector(unsigned TypeIdx);
271 /// True iff the specified type index is a pointer (with any address space).
272 LegalityPredicate isPointer(unsigned TypeIdx);
273 /// True iff the specified type index is a pointer with the specified address
274 /// space.
275 LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace);
276
277 /// True if the type index is a vector with element type \p EltTy
278 LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy);
279
280 /// True iff the specified type index is a scalar that's narrower than the given
281 /// size.
282 LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size);
283
284 /// True iff the specified type index is a scalar that's wider than the given
285 /// size.
286 LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size);
287
288 /// True iff the specified type index is a scalar or vector with an element type
289 /// that's narrower than the given size.
290 LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size);
291
292 /// True iff the specified type index is a scalar or a vector with an element
293 /// type that's wider than the given size.
294 LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size);
295
296 /// True iff the specified type index is a scalar whose size is not a power of
297 /// 2.
298 LegalityPredicate sizeNotPow2(unsigned TypeIdx);
299
300 /// True iff the specified type index is a scalar or vector whose element size
301 /// is not a power of 2.
302 LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx);
303
304 /// True if the total bitwidth of the specified type index is \p Size bits.
305 LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size);
306
307 /// True iff the specified type indices are both the same bit size.
308 LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1);
309
310 /// True iff the first type index has a larger total bit size than second type
311 /// index.
312 LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1);
313
314 /// True iff the first type index has a smaller total bit size than second type
315 /// index.
316 LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1);
317
318 /// True iff the specified MMO index has a size that is not a power of 2
319 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
320 /// True iff the specified type index is a vector whose element count is not a
321 /// power of 2.
322 LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
323 /// True iff the specified MMO index has at an atomic ordering of at Ordering or
324 /// stronger.
325 LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
326 AtomicOrdering Ordering);
327 } // end namespace LegalityPredicates
328
329 namespace LegalizeMutations {
330 /// Select this specific type for the given type index.
331 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
332
333 /// Keep the same type as the given type index.
334 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx);
335
336 /// Keep the same scalar or element type as the given type index.
337 LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
338
339 /// Keep the same scalar or element type as the given type.
340 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
341
342 /// Change the scalar size or element size to have the same scalar size as type
343 /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and
344 /// only changes the size.
345 LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx);
346
347 /// Widen the scalar type or vector element type for the given type index to the
348 /// next power of 2.
349 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
350
351 /// Add more elements to the type for the given type index to the next power of
352 /// 2.
353 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0);
354 /// Break up the vector type for the given type index into the element type.
355 LegalizeMutation scalarize(unsigned TypeIdx);
356 } // end namespace LegalizeMutations
357
358 /// A single rule in a legalizer info ruleset.
359 /// The specified action is chosen when the predicate is true. Where appropriate
360 /// for the action (e.g. for WidenScalar) the new type is selected using the
361 /// given mutator.
362 class LegalizeRule {
363 LegalityPredicate Predicate;
364 LegalizeAction Action;
365 LegalizeMutation Mutation;
366
367 public:
368 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action,
369 LegalizeMutation Mutation = nullptr)
Predicate(Predicate)370 : Predicate(Predicate), Action(Action), Mutation(Mutation) {}
371
372 /// Test whether the LegalityQuery matches.
match(const LegalityQuery & Query)373 bool match(const LegalityQuery &Query) const {
374 return Predicate(Query);
375 }
376
getAction()377 LegalizeAction getAction() const { return Action; }
378
379 /// Determine the change to make.
determineMutation(const LegalityQuery & Query)380 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const {
381 if (Mutation)
382 return Mutation(Query);
383 return std::make_pair(0, LLT{});
384 }
385 };
386
387 class LegalizeRuleSet {
388 /// When non-zero, the opcode we are an alias of
389 unsigned AliasOf;
390 /// If true, there is another opcode that aliases this one
391 bool IsAliasedByAnother;
392 SmallVector<LegalizeRule, 2> Rules;
393
394 #ifndef NDEBUG
395 /// If bit I is set, this rule set contains a rule that may handle (predicate
396 /// or perform an action upon (or both)) the type index I. The uncertainty
397 /// comes from free-form rules executing user-provided lambda functions. We
398 /// conservatively assume such rules do the right thing and cover all type
399 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs
400 /// to be to distinguish such cases from the cases where all type indices are
401 /// individually handled.
402 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC -
403 MCOI::OPERAND_FIRST_GENERIC + 2};
404 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM -
405 MCOI::OPERAND_FIRST_GENERIC_IMM + 2};
406 #endif
407
typeIdx(unsigned TypeIdx)408 unsigned typeIdx(unsigned TypeIdx) {
409 assert(TypeIdx <=
410 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) &&
411 "Type Index is out of bounds");
412 #ifndef NDEBUG
413 TypeIdxsCovered.set(TypeIdx);
414 #endif
415 return TypeIdx;
416 }
417
immIdx(unsigned ImmIdx)418 unsigned immIdx(unsigned ImmIdx) {
419 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM -
420 MCOI::OPERAND_FIRST_GENERIC_IMM) &&
421 "Imm Index is out of bounds");
422 #ifndef NDEBUG
423 ImmIdxsCovered.set(ImmIdx);
424 #endif
425 return ImmIdx;
426 }
427
markAllIdxsAsCovered()428 void markAllIdxsAsCovered() {
429 #ifndef NDEBUG
430 TypeIdxsCovered.set();
431 ImmIdxsCovered.set();
432 #endif
433 }
434
add(const LegalizeRule & Rule)435 void add(const LegalizeRule &Rule) {
436 assert(AliasOf == 0 &&
437 "RuleSet is aliased, change the representative opcode instead");
438 Rules.push_back(Rule);
439 }
440
always(const LegalityQuery &)441 static bool always(const LegalityQuery &) { return true; }
442
443 /// Use the given action when the predicate is true.
444 /// Action should not be an action that requires mutation.
actionIf(LegalizeAction Action,LegalityPredicate Predicate)445 LegalizeRuleSet &actionIf(LegalizeAction Action,
446 LegalityPredicate Predicate) {
447 add({Predicate, Action});
448 return *this;
449 }
450 /// Use the given action when the predicate is true.
451 /// Action should be an action that requires mutation.
actionIf(LegalizeAction Action,LegalityPredicate Predicate,LegalizeMutation Mutation)452 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate,
453 LegalizeMutation Mutation) {
454 add({Predicate, Action, Mutation});
455 return *this;
456 }
457 /// Use the given action when type index 0 is any type in the given list.
458 /// Action should not be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<LLT> Types)459 LegalizeRuleSet &actionFor(LegalizeAction Action,
460 std::initializer_list<LLT> Types) {
461 using namespace LegalityPredicates;
462 return actionIf(Action, typeInSet(typeIdx(0), Types));
463 }
464 /// Use the given action when type index 0 is any type in the given list.
465 /// Action should be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<LLT> Types,LegalizeMutation Mutation)466 LegalizeRuleSet &actionFor(LegalizeAction Action,
467 std::initializer_list<LLT> Types,
468 LegalizeMutation Mutation) {
469 using namespace LegalityPredicates;
470 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation);
471 }
472 /// Use the given action when type indexes 0 and 1 is any type pair in the
473 /// given list.
474 /// Action should not be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types)475 LegalizeRuleSet &actionFor(LegalizeAction Action,
476 std::initializer_list<std::pair<LLT, LLT>> Types) {
477 using namespace LegalityPredicates;
478 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
479 }
480 /// Use the given action when type indexes 0 and 1 is any type pair in the
481 /// given list.
482 /// Action should be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)483 LegalizeRuleSet &actionFor(LegalizeAction Action,
484 std::initializer_list<std::pair<LLT, LLT>> Types,
485 LegalizeMutation Mutation) {
486 using namespace LegalityPredicates;
487 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types),
488 Mutation);
489 }
490 /// Use the given action when type index 0 is any type in the given list and
491 /// imm index 0 is anything. Action should not be an action that requires
492 /// mutation.
actionForTypeWithAnyImm(LegalizeAction Action,std::initializer_list<LLT> Types)493 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action,
494 std::initializer_list<LLT> Types) {
495 using namespace LegalityPredicates;
496 immIdx(0); // Inform verifier imm idx 0 is handled.
497 return actionIf(Action, typeInSet(typeIdx(0), Types));
498 }
499
actionForTypeWithAnyImm(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types)500 LegalizeRuleSet &actionForTypeWithAnyImm(
501 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) {
502 using namespace LegalityPredicates;
503 immIdx(0); // Inform verifier imm idx 0 is handled.
504 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
505 }
506
507 /// Use the given action when type indexes 0 and 1 are both in the given list.
508 /// That is, the type pair is in the cartesian product of the list.
509 /// Action should not be an action that requires mutation.
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types)510 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action,
511 std::initializer_list<LLT> Types) {
512 using namespace LegalityPredicates;
513 return actionIf(Action, all(typeInSet(typeIdx(0), Types),
514 typeInSet(typeIdx(1), Types)));
515 }
516 /// Use the given action when type indexes 0 and 1 are both in their
517 /// respective lists.
518 /// That is, the type pair is in the cartesian product of the lists
519 /// Action should not be an action that requires mutation.
520 LegalizeRuleSet &
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)521 actionForCartesianProduct(LegalizeAction Action,
522 std::initializer_list<LLT> Types0,
523 std::initializer_list<LLT> Types1) {
524 using namespace LegalityPredicates;
525 return actionIf(Action, all(typeInSet(typeIdx(0), Types0),
526 typeInSet(typeIdx(1), Types1)));
527 }
528 /// Use the given action when type indexes 0, 1, and 2 are all in their
529 /// respective lists.
530 /// That is, the type triple is in the cartesian product of the lists
531 /// Action should not be an action that requires mutation.
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)532 LegalizeRuleSet &actionForCartesianProduct(
533 LegalizeAction Action, std::initializer_list<LLT> Types0,
534 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) {
535 using namespace LegalityPredicates;
536 return actionIf(Action, all(typeInSet(typeIdx(0), Types0),
537 all(typeInSet(typeIdx(1), Types1),
538 typeInSet(typeIdx(2), Types2))));
539 }
540
541 public:
LegalizeRuleSet()542 LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false), Rules() {}
543
isAliasedByAnother()544 bool isAliasedByAnother() { return IsAliasedByAnother; }
setIsAliasedByAnother()545 void setIsAliasedByAnother() { IsAliasedByAnother = true; }
aliasTo(unsigned Opcode)546 void aliasTo(unsigned Opcode) {
547 assert((AliasOf == 0 || AliasOf == Opcode) &&
548 "Opcode is already aliased to another opcode");
549 assert(Rules.empty() && "Aliasing will discard rules");
550 AliasOf = Opcode;
551 }
getAlias()552 unsigned getAlias() const { return AliasOf; }
553
554 /// The instruction is legal if predicate is true.
legalIf(LegalityPredicate Predicate)555 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) {
556 // We have no choice but conservatively assume that the free-form
557 // user-provided Predicate properly handles all type indices:
558 markAllIdxsAsCovered();
559 return actionIf(LegalizeAction::Legal, Predicate);
560 }
561 /// The instruction is legal when type index 0 is any type in the given list.
legalFor(std::initializer_list<LLT> Types)562 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) {
563 return actionFor(LegalizeAction::Legal, Types);
564 }
565 /// The instruction is legal when type indexes 0 and 1 is any type pair in the
566 /// given list.
legalFor(std::initializer_list<std::pair<LLT,LLT>> Types)567 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
568 return actionFor(LegalizeAction::Legal, Types);
569 }
570 /// The instruction is legal when type index 0 is any type in the given list
571 /// and imm index 0 is anything.
legalForTypeWithAnyImm(std::initializer_list<LLT> Types)572 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
573 markAllIdxsAsCovered();
574 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types);
575 }
576
legalForTypeWithAnyImm(std::initializer_list<std::pair<LLT,LLT>> Types)577 LegalizeRuleSet &legalForTypeWithAnyImm(
578 std::initializer_list<std::pair<LLT, LLT>> Types) {
579 markAllIdxsAsCovered();
580 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types);
581 }
582
583 /// The instruction is legal when type indexes 0 and 1 along with the memory
584 /// size and minimum alignment is any type and size tuple in the given list.
legalForTypesWithMemDesc(std::initializer_list<LegalityPredicates::TypePairAndMemDesc> TypesAndMemDesc)585 LegalizeRuleSet &legalForTypesWithMemDesc(
586 std::initializer_list<LegalityPredicates::TypePairAndMemDesc>
587 TypesAndMemDesc) {
588 return actionIf(LegalizeAction::Legal,
589 LegalityPredicates::typePairAndMemDescInSet(
590 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc));
591 }
592 /// The instruction is legal when type indexes 0 and 1 are both in the given
593 /// list. That is, the type pair is in the cartesian product of the list.
legalForCartesianProduct(std::initializer_list<LLT> Types)594 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) {
595 return actionForCartesianProduct(LegalizeAction::Legal, Types);
596 }
597 /// The instruction is legal when type indexes 0 and 1 are both their
598 /// respective lists.
legalForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)599 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
600 std::initializer_list<LLT> Types1) {
601 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1);
602 }
603 /// The instruction is legal when type indexes 0, 1, and 2 are both their
604 /// respective lists.
legalForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)605 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
606 std::initializer_list<LLT> Types1,
607 std::initializer_list<LLT> Types2) {
608 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1,
609 Types2);
610 }
611
alwaysLegal()612 LegalizeRuleSet &alwaysLegal() {
613 using namespace LegalizeMutations;
614 markAllIdxsAsCovered();
615 return actionIf(LegalizeAction::Legal, always);
616 }
617
618 /// The specified type index is coerced if predicate is true.
bitcastIf(LegalityPredicate Predicate,LegalizeMutation Mutation)619 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate,
620 LegalizeMutation Mutation) {
621 // We have no choice but conservatively assume that lowering with a
622 // free-form user provided Predicate properly handles all type indices:
623 markAllIdxsAsCovered();
624 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation);
625 }
626
627 /// The instruction is lowered.
lower()628 LegalizeRuleSet &lower() {
629 using namespace LegalizeMutations;
630 // We have no choice but conservatively assume that predicate-less lowering
631 // properly handles all type indices by design:
632 markAllIdxsAsCovered();
633 return actionIf(LegalizeAction::Lower, always);
634 }
635 /// The instruction is lowered if predicate is true. Keep type index 0 as the
636 /// same type.
lowerIf(LegalityPredicate Predicate)637 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) {
638 using namespace LegalizeMutations;
639 // We have no choice but conservatively assume that lowering with a
640 // free-form user provided Predicate properly handles all type indices:
641 markAllIdxsAsCovered();
642 return actionIf(LegalizeAction::Lower, Predicate);
643 }
644 /// The instruction is lowered if predicate is true.
lowerIf(LegalityPredicate Predicate,LegalizeMutation Mutation)645 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate,
646 LegalizeMutation Mutation) {
647 // We have no choice but conservatively assume that lowering with a
648 // free-form user provided Predicate properly handles all type indices:
649 markAllIdxsAsCovered();
650 return actionIf(LegalizeAction::Lower, Predicate, Mutation);
651 }
652 /// The instruction is lowered when type index 0 is any type in the given
653 /// list. Keep type index 0 as the same type.
lowerFor(std::initializer_list<LLT> Types)654 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) {
655 return actionFor(LegalizeAction::Lower, Types);
656 }
657 /// The instruction is lowered when type index 0 is any type in the given
658 /// list.
lowerFor(std::initializer_list<LLT> Types,LegalizeMutation Mutation)659 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types,
660 LegalizeMutation Mutation) {
661 return actionFor(LegalizeAction::Lower, Types, Mutation);
662 }
663 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
664 /// the given list. Keep type index 0 as the same type.
lowerFor(std::initializer_list<std::pair<LLT,LLT>> Types)665 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
666 return actionFor(LegalizeAction::Lower, Types);
667 }
668 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
669 /// the given list.
lowerFor(std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)670 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types,
671 LegalizeMutation Mutation) {
672 return actionFor(LegalizeAction::Lower, Types, Mutation);
673 }
674 /// The instruction is lowered when type indexes 0 and 1 are both in their
675 /// respective lists.
lowerForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)676 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
677 std::initializer_list<LLT> Types1) {
678 using namespace LegalityPredicates;
679 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1);
680 }
681 /// The instruction is lowered when when type indexes 0, 1, and 2 are all in
682 /// their respective lists.
lowerForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)683 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
684 std::initializer_list<LLT> Types1,
685 std::initializer_list<LLT> Types2) {
686 using namespace LegalityPredicates;
687 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1,
688 Types2);
689 }
690
691 /// The instruction is emitted as a library call.
libcall()692 LegalizeRuleSet &libcall() {
693 using namespace LegalizeMutations;
694 // We have no choice but conservatively assume that predicate-less lowering
695 // properly handles all type indices by design:
696 markAllIdxsAsCovered();
697 return actionIf(LegalizeAction::Libcall, always);
698 }
699
700 /// Like legalIf, but for the Libcall action.
libcallIf(LegalityPredicate Predicate)701 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
702 // We have no choice but conservatively assume that a libcall with a
703 // free-form user provided Predicate properly handles all type indices:
704 markAllIdxsAsCovered();
705 return actionIf(LegalizeAction::Libcall, Predicate);
706 }
libcallFor(std::initializer_list<LLT> Types)707 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
708 return actionFor(LegalizeAction::Libcall, Types);
709 }
710 LegalizeRuleSet &
libcallFor(std::initializer_list<std::pair<LLT,LLT>> Types)711 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
712 return actionFor(LegalizeAction::Libcall, Types);
713 }
714 LegalizeRuleSet &
libcallForCartesianProduct(std::initializer_list<LLT> Types)715 libcallForCartesianProduct(std::initializer_list<LLT> Types) {
716 return actionForCartesianProduct(LegalizeAction::Libcall, Types);
717 }
718 LegalizeRuleSet &
libcallForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)719 libcallForCartesianProduct(std::initializer_list<LLT> Types0,
720 std::initializer_list<LLT> Types1) {
721 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1);
722 }
723
724 /// Widen the scalar to the one selected by the mutation if the predicate is
725 /// true.
widenScalarIf(LegalityPredicate Predicate,LegalizeMutation Mutation)726 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate,
727 LegalizeMutation Mutation) {
728 // We have no choice but conservatively assume that an action with a
729 // free-form user provided Predicate properly handles all type indices:
730 markAllIdxsAsCovered();
731 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation);
732 }
733 /// Narrow the scalar to the one selected by the mutation if the predicate is
734 /// true.
narrowScalarIf(LegalityPredicate Predicate,LegalizeMutation Mutation)735 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate,
736 LegalizeMutation Mutation) {
737 // We have no choice but conservatively assume that an action with a
738 // free-form user provided Predicate properly handles all type indices:
739 markAllIdxsAsCovered();
740 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation);
741 }
742 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any
743 /// type pair in the given list.
744 LegalizeRuleSet &
narrowScalarFor(std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)745 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types,
746 LegalizeMutation Mutation) {
747 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation);
748 }
749
750 /// Add more elements to reach the type selected by the mutation if the
751 /// predicate is true.
moreElementsIf(LegalityPredicate Predicate,LegalizeMutation Mutation)752 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate,
753 LegalizeMutation Mutation) {
754 // We have no choice but conservatively assume that an action with a
755 // free-form user provided Predicate properly handles all type indices:
756 markAllIdxsAsCovered();
757 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation);
758 }
759 /// Remove elements to reach the type selected by the mutation if the
760 /// predicate is true.
fewerElementsIf(LegalityPredicate Predicate,LegalizeMutation Mutation)761 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate,
762 LegalizeMutation Mutation) {
763 // We have no choice but conservatively assume that an action with a
764 // free-form user provided Predicate properly handles all type indices:
765 markAllIdxsAsCovered();
766 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation);
767 }
768
769 /// The instruction is unsupported.
unsupported()770 LegalizeRuleSet &unsupported() {
771 markAllIdxsAsCovered();
772 return actionIf(LegalizeAction::Unsupported, always);
773 }
unsupportedIf(LegalityPredicate Predicate)774 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) {
775 return actionIf(LegalizeAction::Unsupported, Predicate);
776 }
777
unsupportedFor(std::initializer_list<LLT> Types)778 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) {
779 return actionFor(LegalizeAction::Unsupported, Types);
780 }
781
unsupportedIfMemSizeNotPow2()782 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() {
783 return actionIf(LegalizeAction::Unsupported,
784 LegalityPredicates::memSizeInBytesNotPow2(0));
785 }
lowerIfMemSizeNotPow2()786 LegalizeRuleSet &lowerIfMemSizeNotPow2() {
787 return actionIf(LegalizeAction::Lower,
788 LegalityPredicates::memSizeInBytesNotPow2(0));
789 }
790
customIf(LegalityPredicate Predicate)791 LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
792 // We have no choice but conservatively assume that a custom action with a
793 // free-form user provided Predicate properly handles all type indices:
794 markAllIdxsAsCovered();
795 return actionIf(LegalizeAction::Custom, Predicate);
796 }
customFor(std::initializer_list<LLT> Types)797 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
798 return actionFor(LegalizeAction::Custom, Types);
799 }
800
801 /// The instruction is custom when type indexes 0 and 1 is any type pair in the
802 /// given list.
customFor(std::initializer_list<std::pair<LLT,LLT>> Types)803 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
804 return actionFor(LegalizeAction::Custom, Types);
805 }
806
customForCartesianProduct(std::initializer_list<LLT> Types)807 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
808 return actionForCartesianProduct(LegalizeAction::Custom, Types);
809 }
810 LegalizeRuleSet &
customForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)811 customForCartesianProduct(std::initializer_list<LLT> Types0,
812 std::initializer_list<LLT> Types1) {
813 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1);
814 }
815
816 /// Unconditionally custom lower.
custom()817 LegalizeRuleSet &custom() {
818 return customIf(always);
819 }
820
821 /// Widen the scalar to the next power of two that is at least MinSize.
822 /// No effect if the type is not a scalar or is a power of two.
823 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx,
824 unsigned MinSize = 0) {
825 using namespace LegalityPredicates;
826 return actionIf(
827 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)),
828 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
829 }
830
831 /// Widen the scalar or vector element type to the next power of two that is
832 /// at least MinSize. No effect if the scalar size is a power of two.
833 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx,
834 unsigned MinSize = 0) {
835 using namespace LegalityPredicates;
836 return actionIf(
837 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)),
838 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
839 }
840
narrowScalar(unsigned TypeIdx,LegalizeMutation Mutation)841 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) {
842 using namespace LegalityPredicates;
843 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)),
844 Mutation);
845 }
846
scalarize(unsigned TypeIdx)847 LegalizeRuleSet &scalarize(unsigned TypeIdx) {
848 using namespace LegalityPredicates;
849 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)),
850 LegalizeMutations::scalarize(TypeIdx));
851 }
852
scalarizeIf(LegalityPredicate Predicate,unsigned TypeIdx)853 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) {
854 using namespace LegalityPredicates;
855 return actionIf(LegalizeAction::FewerElements,
856 all(Predicate, isVector(typeIdx(TypeIdx))),
857 LegalizeMutations::scalarize(TypeIdx));
858 }
859
860 /// Ensure the scalar or element is at least as wide as Ty.
minScalarOrElt(unsigned TypeIdx,const LLT Ty)861 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) {
862 using namespace LegalityPredicates;
863 using namespace LegalizeMutations;
864 return actionIf(LegalizeAction::WidenScalar,
865 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()),
866 changeElementTo(typeIdx(TypeIdx), Ty));
867 }
868
869 /// Ensure the scalar or element is at least as wide as Ty.
minScalarOrEltIf(LegalityPredicate Predicate,unsigned TypeIdx,const LLT Ty)870 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate,
871 unsigned TypeIdx, const LLT Ty) {
872 using namespace LegalityPredicates;
873 using namespace LegalizeMutations;
874 return actionIf(LegalizeAction::WidenScalar,
875 all(Predicate, scalarOrEltNarrowerThan(
876 TypeIdx, Ty.getScalarSizeInBits())),
877 changeElementTo(typeIdx(TypeIdx), Ty));
878 }
879
880 /// Ensure the scalar is at least as wide as Ty.
minScalar(unsigned TypeIdx,const LLT Ty)881 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) {
882 using namespace LegalityPredicates;
883 using namespace LegalizeMutations;
884 return actionIf(LegalizeAction::WidenScalar,
885 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()),
886 changeTo(typeIdx(TypeIdx), Ty));
887 }
888
889 /// Ensure the scalar is at most as wide as Ty.
maxScalarOrElt(unsigned TypeIdx,const LLT Ty)890 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) {
891 using namespace LegalityPredicates;
892 using namespace LegalizeMutations;
893 return actionIf(LegalizeAction::NarrowScalar,
894 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()),
895 changeElementTo(typeIdx(TypeIdx), Ty));
896 }
897
898 /// Ensure the scalar is at most as wide as Ty.
maxScalar(unsigned TypeIdx,const LLT Ty)899 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) {
900 using namespace LegalityPredicates;
901 using namespace LegalizeMutations;
902 return actionIf(LegalizeAction::NarrowScalar,
903 scalarWiderThan(TypeIdx, Ty.getSizeInBits()),
904 changeTo(typeIdx(TypeIdx), Ty));
905 }
906
907 /// Conditionally limit the maximum size of the scalar.
908 /// For example, when the maximum size of one type depends on the size of
909 /// another such as extracting N bits from an M bit container.
maxScalarIf(LegalityPredicate Predicate,unsigned TypeIdx,const LLT Ty)910 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
911 const LLT Ty) {
912 using namespace LegalityPredicates;
913 using namespace LegalizeMutations;
914 return actionIf(
915 LegalizeAction::NarrowScalar,
916 [=](const LegalityQuery &Query) {
917 const LLT QueryTy = Query.Types[TypeIdx];
918 return QueryTy.isScalar() &&
919 QueryTy.getSizeInBits() > Ty.getSizeInBits() &&
920 Predicate(Query);
921 },
922 changeElementTo(typeIdx(TypeIdx), Ty));
923 }
924
925 /// Limit the range of scalar sizes to MinTy and MaxTy.
clampScalar(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)926 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy,
927 const LLT MaxTy) {
928 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types");
929 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy);
930 }
931
932 /// Limit the range of scalar sizes to MinTy and MaxTy.
clampScalarOrElt(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)933 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy,
934 const LLT MaxTy) {
935 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy);
936 }
937
938 /// Widen the scalar to match the size of another.
minScalarSameAs(unsigned TypeIdx,unsigned LargeTypeIdx)939 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) {
940 typeIdx(TypeIdx);
941 return widenScalarIf(
942 [=](const LegalityQuery &Query) {
943 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
944 Query.Types[TypeIdx].getSizeInBits();
945 },
946 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx));
947 }
948
949 /// Narrow the scalar to match the size of another.
maxScalarSameAs(unsigned TypeIdx,unsigned NarrowTypeIdx)950 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) {
951 typeIdx(TypeIdx);
952 return narrowScalarIf(
953 [=](const LegalityQuery &Query) {
954 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() <
955 Query.Types[TypeIdx].getSizeInBits();
956 },
957 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx));
958 }
959
960 /// Change the type \p TypeIdx to have the same scalar size as type \p
961 /// SameSizeIdx.
scalarSameSizeAs(unsigned TypeIdx,unsigned SameSizeIdx)962 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) {
963 return minScalarSameAs(TypeIdx, SameSizeIdx)
964 .maxScalarSameAs(TypeIdx, SameSizeIdx);
965 }
966
967 /// Conditionally widen the scalar or elt to match the size of another.
minScalarEltSameAsIf(LegalityPredicate Predicate,unsigned TypeIdx,unsigned LargeTypeIdx)968 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate,
969 unsigned TypeIdx, unsigned LargeTypeIdx) {
970 typeIdx(TypeIdx);
971 return widenScalarIf(
972 [=](const LegalityQuery &Query) {
973 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
974 Query.Types[TypeIdx].getScalarSizeInBits() &&
975 Predicate(Query);
976 },
977 [=](const LegalityQuery &Query) {
978 LLT T = Query.Types[LargeTypeIdx];
979 return std::make_pair(TypeIdx, T);
980 });
981 }
982
983 /// Conditionally narrow the scalar or elt to match the size of another.
maxScalarEltSameAsIf(LegalityPredicate Predicate,unsigned TypeIdx,unsigned SmallTypeIdx)984 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate,
985 unsigned TypeIdx,
986 unsigned SmallTypeIdx) {
987 typeIdx(TypeIdx);
988 return narrowScalarIf(
989 [=](const LegalityQuery &Query) {
990 return Query.Types[SmallTypeIdx].getScalarSizeInBits() <
991 Query.Types[TypeIdx].getScalarSizeInBits() &&
992 Predicate(Query);
993 },
994 [=](const LegalityQuery &Query) {
995 LLT T = Query.Types[SmallTypeIdx];
996 return std::make_pair(TypeIdx, T);
997 });
998 }
999
1000 /// Add more elements to the vector to reach the next power of two.
1001 /// No effect if the type is not a vector or the element count is a power of
1002 /// two.
moreElementsToNextPow2(unsigned TypeIdx)1003 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) {
1004 using namespace LegalityPredicates;
1005 return actionIf(LegalizeAction::MoreElements,
1006 numElementsNotPow2(typeIdx(TypeIdx)),
1007 LegalizeMutations::moreElementsToNextPow2(TypeIdx));
1008 }
1009
1010 /// Limit the number of elements in EltTy vectors to at least MinElements.
clampMinNumElements(unsigned TypeIdx,const LLT EltTy,unsigned MinElements)1011 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy,
1012 unsigned MinElements) {
1013 // Mark the type index as covered:
1014 typeIdx(TypeIdx);
1015 return actionIf(
1016 LegalizeAction::MoreElements,
1017 [=](const LegalityQuery &Query) {
1018 LLT VecTy = Query.Types[TypeIdx];
1019 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1020 VecTy.getNumElements() < MinElements;
1021 },
1022 [=](const LegalityQuery &Query) {
1023 LLT VecTy = Query.Types[TypeIdx];
1024 return std::make_pair(
1025 TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType()));
1026 });
1027 }
1028 /// Limit the number of elements in EltTy vectors to at most MaxElements.
clampMaxNumElements(unsigned TypeIdx,const LLT EltTy,unsigned MaxElements)1029 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy,
1030 unsigned MaxElements) {
1031 // Mark the type index as covered:
1032 typeIdx(TypeIdx);
1033 return actionIf(
1034 LegalizeAction::FewerElements,
1035 [=](const LegalityQuery &Query) {
1036 LLT VecTy = Query.Types[TypeIdx];
1037 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1038 VecTy.getNumElements() > MaxElements;
1039 },
1040 [=](const LegalityQuery &Query) {
1041 LLT VecTy = Query.Types[TypeIdx];
1042 LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements),
1043 VecTy.getElementType());
1044 return std::make_pair(TypeIdx, NewTy);
1045 });
1046 }
1047 /// Limit the number of elements for the given vectors to at least MinTy's
1048 /// number of elements and at most MaxTy's number of elements.
1049 ///
1050 /// No effect if the type is not a vector or does not have the same element
1051 /// type as the constraints.
1052 /// The element type of MinTy and MaxTy must match.
clampNumElements(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)1053 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy,
1054 const LLT MaxTy) {
1055 assert(MinTy.getElementType() == MaxTy.getElementType() &&
1056 "Expected element types to agree");
1057
1058 const LLT EltTy = MinTy.getElementType();
1059 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements())
1060 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements());
1061 }
1062
1063 /// Fallback on the previous implementation. This should only be used while
1064 /// porting a rule.
fallback()1065 LegalizeRuleSet &fallback() {
1066 add({always, LegalizeAction::UseLegacyRules});
1067 return *this;
1068 }
1069
1070 /// Check if there is no type index which is obviously not handled by the
1071 /// LegalizeRuleSet in any way at all.
1072 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1073 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const;
1074 /// Check if there is no imm index which is obviously not handled by the
1075 /// LegalizeRuleSet in any way at all.
1076 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1077 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const;
1078
1079 /// Apply the ruleset to the given LegalityQuery.
1080 LegalizeActionStep apply(const LegalityQuery &Query) const;
1081 };
1082
1083 class LegalizerInfo {
1084 public:
1085 virtual ~LegalizerInfo() = default;
1086
getLegacyLegalizerInfo()1087 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const {
1088 return LegacyInfo;
1089 }
getLegacyLegalizerInfo()1090 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; }
1091
1092 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const;
1093 unsigned getActionDefinitionsIdx(unsigned Opcode) const;
1094
1095 /// Perform simple self-diagnostic and assert if there is anything obviously
1096 /// wrong with the actions set up.
1097 void verify(const MCInstrInfo &MII) const;
1098
1099 /// Get the action definitions for the given opcode. Use this to run a
1100 /// LegalityQuery through the definitions.
1101 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const;
1102
1103 /// Get the action definition builder for the given opcode. Use this to define
1104 /// the action definitions.
1105 ///
1106 /// It is an error to request an opcode that has already been requested by the
1107 /// multiple-opcode variant.
1108 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode);
1109
1110 /// Get the action definition builder for the given set of opcodes. Use this
1111 /// to define the action definitions for multiple opcodes at once. The first
1112 /// opcode given will be considered the representative opcode and will hold
1113 /// the definitions whereas the other opcodes will be configured to refer to
1114 /// the representative opcode. This lowers memory requirements and very
1115 /// slightly improves performance.
1116 ///
1117 /// It would be very easy to introduce unexpected side-effects as a result of
1118 /// this aliasing if it were permitted to request different but intersecting
1119 /// sets of opcodes but that is difficult to keep track of. It is therefore an
1120 /// error to request the same opcode twice using this API, to request an
1121 /// opcode that already has definitions, or to use the single-opcode API on an
1122 /// opcode that has already been requested by this API.
1123 LegalizeRuleSet &
1124 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes);
1125 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom);
1126
1127 /// Determine what action should be taken to legalize the described
1128 /// instruction. Requires computeTables to have been called.
1129 ///
1130 /// \returns a description of the next legalization step to perform.
1131 LegalizeActionStep getAction(const LegalityQuery &Query) const;
1132
1133 /// Determine what action should be taken to legalize the given generic
1134 /// instruction.
1135 ///
1136 /// \returns a description of the next legalization step to perform.
1137 LegalizeActionStep getAction(const MachineInstr &MI,
1138 const MachineRegisterInfo &MRI) const;
1139
isLegal(const LegalityQuery & Query)1140 bool isLegal(const LegalityQuery &Query) const {
1141 return getAction(Query).Action == LegalizeAction::Legal;
1142 }
1143
isLegalOrCustom(const LegalityQuery & Query)1144 bool isLegalOrCustom(const LegalityQuery &Query) const {
1145 auto Action = getAction(Query).Action;
1146 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom;
1147 }
1148
1149 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
1150 bool isLegalOrCustom(const MachineInstr &MI,
1151 const MachineRegisterInfo &MRI) const;
1152
1153 /// Called for instructions with the Custom LegalizationAction.
legalizeCustom(LegalizerHelper & Helper,MachineInstr & MI)1154 virtual bool legalizeCustom(LegalizerHelper &Helper,
1155 MachineInstr &MI) const {
1156 llvm_unreachable("must implement this if custom action is used");
1157 }
1158
1159 /// \returns true if MI is either legal or has been legalized and false if not
1160 /// legal.
1161 /// Return true if MI is either legal or has been legalized and false
1162 /// if not legal.
legalizeIntrinsic(LegalizerHelper & Helper,MachineInstr & MI)1163 virtual bool legalizeIntrinsic(LegalizerHelper &Helper,
1164 MachineInstr &MI) const {
1165 return true;
1166 }
1167
1168 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while
1169 /// widening a constant of type SmallTy which targets can override.
1170 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which
1171 /// will be the default.
1172 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const;
1173
1174 private:
1175 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START;
1176 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END;
1177
1178 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1];
1179 LegacyLegalizerInfo LegacyInfo;
1180 };
1181
1182 #ifndef NDEBUG
1183 /// Checks that MIR is fully legal, returns an illegal instruction if it's not,
1184 /// nullptr otherwise
1185 const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF);
1186 #endif
1187
1188 } // end namespace llvm.
1189
1190 #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
1191