1 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// Interface for Targets to specify which operations they can successfully
10 /// select and how the others should be expanded most efficiently.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
15 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
16
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/TargetOpcodes.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/LowLevelTypeImpl.h"
27 #include "llvm/Support/raw_ostream.h"
28 #include <cassert>
29 #include <cstdint>
30 #include <tuple>
31 #include <unordered_map>
32 #include <utility>
33
34 namespace llvm {
35
36 extern cl::opt<bool> DisableGISelLegalityCheck;
37
38 class LegalizerHelper;
39 class MachineInstr;
40 class MachineRegisterInfo;
41 class MCInstrInfo;
42 class GISelChangeObserver;
43
44 namespace LegalizeActions {
45 enum LegalizeAction : std::uint8_t {
46 /// The operation is expected to be selectable directly by the target, and
47 /// no transformation is necessary.
48 Legal,
49
50 /// The operation should be synthesized from multiple instructions acting on
51 /// a narrower scalar base-type. For example a 64-bit add might be
52 /// implemented in terms of 32-bit add-with-carry.
53 NarrowScalar,
54
55 /// The operation should be implemented in terms of a wider scalar
56 /// base-type. For example a <2 x s8> add could be implemented as a <2
57 /// x s32> add (ignoring the high bits).
58 WidenScalar,
59
60 /// The (vector) operation should be implemented by splitting it into
61 /// sub-vectors where the operation is legal. For example a <8 x s64> add
62 /// might be implemented as 4 separate <2 x s64> adds.
63 FewerElements,
64
65 /// The (vector) operation should be implemented by widening the input
66 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is
67 /// rarely legal, but you might perform an <8 x i8> and then only look at
68 /// the first two results.
69 MoreElements,
70
71 /// Perform the operation on a different, but equivalently sized type.
72 Bitcast,
73
74 /// The operation itself must be expressed in terms of simpler actions on
75 /// this target. E.g. a SREM replaced by an SDIV and subtraction.
76 Lower,
77
78 /// The operation should be implemented as a call to some kind of runtime
79 /// support library. For example this usually happens on machines that don't
80 /// support floating-point operations natively.
81 Libcall,
82
83 /// The target wants to do something special with this combination of
84 /// operand and type. A callback will be issued when it is needed.
85 Custom,
86
87 /// This operation is completely unsupported on the target. A programming
88 /// error has occurred.
89 Unsupported,
90
91 /// Sentinel value for when no action was found in the specified table.
92 NotFound,
93
94 /// Fall back onto the old rules.
95 /// TODO: Remove this once we've migrated
96 UseLegacyRules,
97 };
98 } // end namespace LegalizeActions
99 raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action);
100
101 using LegalizeActions::LegalizeAction;
102
103 /// Legalization is decided based on an instruction's opcode, which type slot
104 /// we're considering, and what the existing type is. These aspects are gathered
105 /// together for convenience in the InstrAspect class.
106 struct InstrAspect {
107 unsigned Opcode;
108 unsigned Idx = 0;
109 LLT Type;
110
InstrAspectInstrAspect111 InstrAspect(unsigned Opcode, LLT Type) : Opcode(Opcode), Type(Type) {}
InstrAspectInstrAspect112 InstrAspect(unsigned Opcode, unsigned Idx, LLT Type)
113 : Opcode(Opcode), Idx(Idx), Type(Type) {}
114
115 bool operator==(const InstrAspect &RHS) const {
116 return Opcode == RHS.Opcode && Idx == RHS.Idx && Type == RHS.Type;
117 }
118 };
119
120 /// The LegalityQuery object bundles together all the information that's needed
121 /// to decide whether a given operation is legal or not.
122 /// For efficiency, it doesn't make a copy of Types so care must be taken not
123 /// to free it before using the query.
124 struct LegalityQuery {
125 unsigned Opcode;
126 ArrayRef<LLT> Types;
127
128 struct MemDesc {
129 uint64_t SizeInBits;
130 uint64_t AlignInBits;
131 AtomicOrdering Ordering;
132 };
133
134 /// Operations which require memory can use this to place requirements on the
135 /// memory type for each MMO.
136 ArrayRef<MemDesc> MMODescrs;
137
LegalityQueryLegalityQuery138 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
139 const ArrayRef<MemDesc> MMODescrs)
140 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
LegalityQueryLegalityQuery141 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
142 : LegalityQuery(Opcode, Types, {}) {}
143
144 raw_ostream &print(raw_ostream &OS) const;
145 };
146
147 /// The result of a query. It either indicates a final answer of Legal or
148 /// Unsupported or describes an action that must be taken to make an operation
149 /// more legal.
150 struct LegalizeActionStep {
151 /// The action to take or the final answer.
152 LegalizeAction Action;
153 /// If describing an action, the type index to change. Otherwise zero.
154 unsigned TypeIdx;
155 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}.
156 LLT NewType;
157
LegalizeActionStepLegalizeActionStep158 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx,
159 const LLT NewType)
160 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {}
161
162 bool operator==(const LegalizeActionStep &RHS) const {
163 return std::tie(Action, TypeIdx, NewType) ==
164 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType);
165 }
166 };
167
168 using LegalityPredicate = std::function<bool (const LegalityQuery &)>;
169 using LegalizeMutation =
170 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
171
172 namespace LegalityPredicates {
173 struct TypePairAndMemDesc {
174 LLT Type0;
175 LLT Type1;
176 uint64_t MemSize;
177 uint64_t Align;
178
179 bool operator==(const TypePairAndMemDesc &Other) const {
180 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
181 Align == Other.Align &&
182 MemSize == Other.MemSize;
183 }
184
185 /// \returns true if this memory access is legal with for the access described
186 /// by \p Other (The alignment is sufficient for the size and result type).
isCompatibleTypePairAndMemDesc187 bool isCompatible(const TypePairAndMemDesc &Other) const {
188 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
189 Align >= Other.Align &&
190 MemSize == Other.MemSize;
191 }
192 };
193
194 /// True iff P0 and P1 are true.
195 template<typename Predicate>
all(Predicate P0,Predicate P1)196 Predicate all(Predicate P0, Predicate P1) {
197 return [=](const LegalityQuery &Query) {
198 return P0(Query) && P1(Query);
199 };
200 }
201 /// True iff all given predicates are true.
202 template<typename Predicate, typename... Args>
all(Predicate P0,Predicate P1,Args...args)203 Predicate all(Predicate P0, Predicate P1, Args... args) {
204 return all(all(P0, P1), args...);
205 }
206
207 /// True iff P0 or P1 are true.
208 template<typename Predicate>
any(Predicate P0,Predicate P1)209 Predicate any(Predicate P0, Predicate P1) {
210 return [=](const LegalityQuery &Query) {
211 return P0(Query) || P1(Query);
212 };
213 }
214 /// True iff any given predicates are true.
215 template<typename Predicate, typename... Args>
any(Predicate P0,Predicate P1,Args...args)216 Predicate any(Predicate P0, Predicate P1, Args... args) {
217 return any(any(P0, P1), args...);
218 }
219
220 /// True iff the given type index is the specified type.
221 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
222 /// True iff the given type index is one of the specified types.
223 LegalityPredicate typeInSet(unsigned TypeIdx,
224 std::initializer_list<LLT> TypesInit);
225
226 /// True iff the given type index is not the specified type.
typeIsNot(unsigned TypeIdx,LLT Type)227 inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
228 return [=](const LegalityQuery &Query) {
229 return Query.Types[TypeIdx] != Type;
230 };
231 }
232
233 /// True iff the given types for the given pair of type indexes is one of the
234 /// specified type pairs.
235 LegalityPredicate
236 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
237 std::initializer_list<std::pair<LLT, LLT>> TypesInit);
238 /// True iff the given types for the given pair of type indexes is one of the
239 /// specified type pairs.
240 LegalityPredicate typePairAndMemDescInSet(
241 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
242 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit);
243 /// True iff the specified type index is a scalar.
244 LegalityPredicate isScalar(unsigned TypeIdx);
245 /// True iff the specified type index is a vector.
246 LegalityPredicate isVector(unsigned TypeIdx);
247 /// True iff the specified type index is a pointer (with any address space).
248 LegalityPredicate isPointer(unsigned TypeIdx);
249 /// True iff the specified type index is a pointer with the specified address
250 /// space.
251 LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace);
252
253 /// True if the type index is a vector with element type \p EltTy
254 LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy);
255
256 /// True iff the specified type index is a scalar that's narrower than the given
257 /// size.
258 LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size);
259
260 /// True iff the specified type index is a scalar that's wider than the given
261 /// size.
262 LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size);
263
264 /// True iff the specified type index is a scalar or vector with an element type
265 /// that's narrower than the given size.
266 LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size);
267
268 /// True iff the specified type index is a scalar or a vector with an element
269 /// type that's wider than the given size.
270 LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size);
271
272 /// True iff the specified type index is a scalar whose size is not a power of
273 /// 2.
274 LegalityPredicate sizeNotPow2(unsigned TypeIdx);
275
276 /// True iff the specified type index is a scalar or vector whose element size
277 /// is not a power of 2.
278 LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx);
279
280 /// True if the total bitwidth of the specified type index is \p Size bits.
281 LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size);
282
283 /// True iff the specified type indices are both the same bit size.
284 LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1);
285
286 /// True iff the first type index has a larger total bit size than second type
287 /// index.
288 LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1);
289
290 /// True iff the first type index has a smaller total bit size than second type
291 /// index.
292 LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1);
293
294 /// True iff the specified MMO index has a size that is not a power of 2
295 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
296 /// True iff the specified type index is a vector whose element count is not a
297 /// power of 2.
298 LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
299 /// True iff the specified MMO index has at an atomic ordering of at Ordering or
300 /// stronger.
301 LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
302 AtomicOrdering Ordering);
303 } // end namespace LegalityPredicates
304
305 namespace LegalizeMutations {
306 /// Select this specific type for the given type index.
307 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
308
309 /// Keep the same type as the given type index.
310 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx);
311
312 /// Keep the same scalar or element type as the given type index.
313 LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
314
315 /// Keep the same scalar or element type as the given type.
316 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
317
318 /// Change the scalar size or element size to have the same scalar size as type
319 /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and
320 /// only changes the size.
321 LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx);
322
323 /// Widen the scalar type or vector element type for the given type index to the
324 /// next power of 2.
325 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
326
327 /// Add more elements to the type for the given type index to the next power of
328 /// 2.
329 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0);
330 /// Break up the vector type for the given type index into the element type.
331 LegalizeMutation scalarize(unsigned TypeIdx);
332 } // end namespace LegalizeMutations
333
334 /// A single rule in a legalizer info ruleset.
335 /// The specified action is chosen when the predicate is true. Where appropriate
336 /// for the action (e.g. for WidenScalar) the new type is selected using the
337 /// given mutator.
338 class LegalizeRule {
339 LegalityPredicate Predicate;
340 LegalizeAction Action;
341 LegalizeMutation Mutation;
342
343 public:
344 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action,
345 LegalizeMutation Mutation = nullptr)
Predicate(Predicate)346 : Predicate(Predicate), Action(Action), Mutation(Mutation) {}
347
348 /// Test whether the LegalityQuery matches.
match(const LegalityQuery & Query)349 bool match(const LegalityQuery &Query) const {
350 return Predicate(Query);
351 }
352
getAction()353 LegalizeAction getAction() const { return Action; }
354
355 /// Determine the change to make.
determineMutation(const LegalityQuery & Query)356 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const {
357 if (Mutation)
358 return Mutation(Query);
359 return std::make_pair(0, LLT{});
360 }
361 };
362
363 class LegalizeRuleSet {
364 /// When non-zero, the opcode we are an alias of
365 unsigned AliasOf;
366 /// If true, there is another opcode that aliases this one
367 bool IsAliasedByAnother;
368 SmallVector<LegalizeRule, 2> Rules;
369
370 #ifndef NDEBUG
371 /// If bit I is set, this rule set contains a rule that may handle (predicate
372 /// or perform an action upon (or both)) the type index I. The uncertainty
373 /// comes from free-form rules executing user-provided lambda functions. We
374 /// conservatively assume such rules do the right thing and cover all type
375 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs
376 /// to be to distinguish such cases from the cases where all type indices are
377 /// individually handled.
378 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC -
379 MCOI::OPERAND_FIRST_GENERIC + 2};
380 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM -
381 MCOI::OPERAND_FIRST_GENERIC_IMM + 2};
382 #endif
383
typeIdx(unsigned TypeIdx)384 unsigned typeIdx(unsigned TypeIdx) {
385 assert(TypeIdx <=
386 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) &&
387 "Type Index is out of bounds");
388 #ifndef NDEBUG
389 TypeIdxsCovered.set(TypeIdx);
390 #endif
391 return TypeIdx;
392 }
393
immIdx(unsigned ImmIdx)394 unsigned immIdx(unsigned ImmIdx) {
395 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM -
396 MCOI::OPERAND_FIRST_GENERIC_IMM) &&
397 "Imm Index is out of bounds");
398 #ifndef NDEBUG
399 ImmIdxsCovered.set(ImmIdx);
400 #endif
401 return ImmIdx;
402 }
403
markAllIdxsAsCovered()404 void markAllIdxsAsCovered() {
405 #ifndef NDEBUG
406 TypeIdxsCovered.set();
407 ImmIdxsCovered.set();
408 #endif
409 }
410
add(const LegalizeRule & Rule)411 void add(const LegalizeRule &Rule) {
412 assert(AliasOf == 0 &&
413 "RuleSet is aliased, change the representative opcode instead");
414 Rules.push_back(Rule);
415 }
416
always(const LegalityQuery &)417 static bool always(const LegalityQuery &) { return true; }
418
419 /// Use the given action when the predicate is true.
420 /// Action should not be an action that requires mutation.
actionIf(LegalizeAction Action,LegalityPredicate Predicate)421 LegalizeRuleSet &actionIf(LegalizeAction Action,
422 LegalityPredicate Predicate) {
423 add({Predicate, Action});
424 return *this;
425 }
426 /// Use the given action when the predicate is true.
427 /// Action should be an action that requires mutation.
actionIf(LegalizeAction Action,LegalityPredicate Predicate,LegalizeMutation Mutation)428 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate,
429 LegalizeMutation Mutation) {
430 add({Predicate, Action, Mutation});
431 return *this;
432 }
433 /// Use the given action when type index 0 is any type in the given list.
434 /// Action should not be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<LLT> Types)435 LegalizeRuleSet &actionFor(LegalizeAction Action,
436 std::initializer_list<LLT> Types) {
437 using namespace LegalityPredicates;
438 return actionIf(Action, typeInSet(typeIdx(0), Types));
439 }
440 /// Use the given action when type index 0 is any type in the given list.
441 /// Action should be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<LLT> Types,LegalizeMutation Mutation)442 LegalizeRuleSet &actionFor(LegalizeAction Action,
443 std::initializer_list<LLT> Types,
444 LegalizeMutation Mutation) {
445 using namespace LegalityPredicates;
446 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation);
447 }
448 /// Use the given action when type indexes 0 and 1 is any type pair in the
449 /// given list.
450 /// Action should not be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types)451 LegalizeRuleSet &actionFor(LegalizeAction Action,
452 std::initializer_list<std::pair<LLT, LLT>> Types) {
453 using namespace LegalityPredicates;
454 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
455 }
456 /// Use the given action when type indexes 0 and 1 is any type pair in the
457 /// given list.
458 /// Action should be an action that requires mutation.
actionFor(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)459 LegalizeRuleSet &actionFor(LegalizeAction Action,
460 std::initializer_list<std::pair<LLT, LLT>> Types,
461 LegalizeMutation Mutation) {
462 using namespace LegalityPredicates;
463 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types),
464 Mutation);
465 }
466 /// Use the given action when type index 0 is any type in the given list and
467 /// imm index 0 is anything. Action should not be an action that requires
468 /// mutation.
actionForTypeWithAnyImm(LegalizeAction Action,std::initializer_list<LLT> Types)469 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action,
470 std::initializer_list<LLT> Types) {
471 using namespace LegalityPredicates;
472 immIdx(0); // Inform verifier imm idx 0 is handled.
473 return actionIf(Action, typeInSet(typeIdx(0), Types));
474 }
475
actionForTypeWithAnyImm(LegalizeAction Action,std::initializer_list<std::pair<LLT,LLT>> Types)476 LegalizeRuleSet &actionForTypeWithAnyImm(
477 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) {
478 using namespace LegalityPredicates;
479 immIdx(0); // Inform verifier imm idx 0 is handled.
480 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
481 }
482
483 /// Use the given action when type indexes 0 and 1 are both in the given list.
484 /// That is, the type pair is in the cartesian product of the list.
485 /// Action should not be an action that requires mutation.
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types)486 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action,
487 std::initializer_list<LLT> Types) {
488 using namespace LegalityPredicates;
489 return actionIf(Action, all(typeInSet(typeIdx(0), Types),
490 typeInSet(typeIdx(1), Types)));
491 }
492 /// Use the given action when type indexes 0 and 1 are both in their
493 /// respective lists.
494 /// That is, the type pair is in the cartesian product of the lists
495 /// Action should not be an action that requires mutation.
496 LegalizeRuleSet &
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)497 actionForCartesianProduct(LegalizeAction Action,
498 std::initializer_list<LLT> Types0,
499 std::initializer_list<LLT> Types1) {
500 using namespace LegalityPredicates;
501 return actionIf(Action, all(typeInSet(typeIdx(0), Types0),
502 typeInSet(typeIdx(1), Types1)));
503 }
504 /// Use the given action when type indexes 0, 1, and 2 are all in their
505 /// respective lists.
506 /// That is, the type triple is in the cartesian product of the lists
507 /// Action should not be an action that requires mutation.
actionForCartesianProduct(LegalizeAction Action,std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)508 LegalizeRuleSet &actionForCartesianProduct(
509 LegalizeAction Action, std::initializer_list<LLT> Types0,
510 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) {
511 using namespace LegalityPredicates;
512 return actionIf(Action, all(typeInSet(typeIdx(0), Types0),
513 all(typeInSet(typeIdx(1), Types1),
514 typeInSet(typeIdx(2), Types2))));
515 }
516
517 public:
LegalizeRuleSet()518 LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false), Rules() {}
519
isAliasedByAnother()520 bool isAliasedByAnother() { return IsAliasedByAnother; }
setIsAliasedByAnother()521 void setIsAliasedByAnother() { IsAliasedByAnother = true; }
aliasTo(unsigned Opcode)522 void aliasTo(unsigned Opcode) {
523 assert((AliasOf == 0 || AliasOf == Opcode) &&
524 "Opcode is already aliased to another opcode");
525 assert(Rules.empty() && "Aliasing will discard rules");
526 AliasOf = Opcode;
527 }
getAlias()528 unsigned getAlias() const { return AliasOf; }
529
530 /// The instruction is legal if predicate is true.
legalIf(LegalityPredicate Predicate)531 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) {
532 // We have no choice but conservatively assume that the free-form
533 // user-provided Predicate properly handles all type indices:
534 markAllIdxsAsCovered();
535 return actionIf(LegalizeAction::Legal, Predicate);
536 }
537 /// The instruction is legal when type index 0 is any type in the given list.
legalFor(std::initializer_list<LLT> Types)538 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) {
539 return actionFor(LegalizeAction::Legal, Types);
540 }
541 /// The instruction is legal when type indexes 0 and 1 is any type pair in the
542 /// given list.
legalFor(std::initializer_list<std::pair<LLT,LLT>> Types)543 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
544 return actionFor(LegalizeAction::Legal, Types);
545 }
546 /// The instruction is legal when type index 0 is any type in the given list
547 /// and imm index 0 is anything.
legalForTypeWithAnyImm(std::initializer_list<LLT> Types)548 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
549 markAllIdxsAsCovered();
550 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types);
551 }
552
legalForTypeWithAnyImm(std::initializer_list<std::pair<LLT,LLT>> Types)553 LegalizeRuleSet &legalForTypeWithAnyImm(
554 std::initializer_list<std::pair<LLT, LLT>> Types) {
555 markAllIdxsAsCovered();
556 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types);
557 }
558
559 /// The instruction is legal when type indexes 0 and 1 along with the memory
560 /// size and minimum alignment is any type and size tuple in the given list.
legalForTypesWithMemDesc(std::initializer_list<LegalityPredicates::TypePairAndMemDesc> TypesAndMemDesc)561 LegalizeRuleSet &legalForTypesWithMemDesc(
562 std::initializer_list<LegalityPredicates::TypePairAndMemDesc>
563 TypesAndMemDesc) {
564 return actionIf(LegalizeAction::Legal,
565 LegalityPredicates::typePairAndMemDescInSet(
566 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc));
567 }
568 /// The instruction is legal when type indexes 0 and 1 are both in the given
569 /// list. That is, the type pair is in the cartesian product of the list.
legalForCartesianProduct(std::initializer_list<LLT> Types)570 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) {
571 return actionForCartesianProduct(LegalizeAction::Legal, Types);
572 }
573 /// The instruction is legal when type indexes 0 and 1 are both their
574 /// respective lists.
legalForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)575 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
576 std::initializer_list<LLT> Types1) {
577 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1);
578 }
579 /// The instruction is legal when type indexes 0, 1, and 2 are both their
580 /// respective lists.
legalForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)581 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
582 std::initializer_list<LLT> Types1,
583 std::initializer_list<LLT> Types2) {
584 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1,
585 Types2);
586 }
587
alwaysLegal()588 LegalizeRuleSet &alwaysLegal() {
589 using namespace LegalizeMutations;
590 markAllIdxsAsCovered();
591 return actionIf(LegalizeAction::Legal, always);
592 }
593
594 /// The specified type index is coerced if predicate is true.
bitcastIf(LegalityPredicate Predicate,LegalizeMutation Mutation)595 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate,
596 LegalizeMutation Mutation) {
597 // We have no choice but conservatively assume that lowering with a
598 // free-form user provided Predicate properly handles all type indices:
599 markAllIdxsAsCovered();
600 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation);
601 }
602
603 /// The instruction is lowered.
lower()604 LegalizeRuleSet &lower() {
605 using namespace LegalizeMutations;
606 // We have no choice but conservatively assume that predicate-less lowering
607 // properly handles all type indices by design:
608 markAllIdxsAsCovered();
609 return actionIf(LegalizeAction::Lower, always);
610 }
611 /// The instruction is lowered if predicate is true. Keep type index 0 as the
612 /// same type.
lowerIf(LegalityPredicate Predicate)613 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) {
614 using namespace LegalizeMutations;
615 // We have no choice but conservatively assume that lowering with a
616 // free-form user provided Predicate properly handles all type indices:
617 markAllIdxsAsCovered();
618 return actionIf(LegalizeAction::Lower, Predicate);
619 }
620 /// The instruction is lowered if predicate is true.
lowerIf(LegalityPredicate Predicate,LegalizeMutation Mutation)621 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate,
622 LegalizeMutation Mutation) {
623 // We have no choice but conservatively assume that lowering with a
624 // free-form user provided Predicate properly handles all type indices:
625 markAllIdxsAsCovered();
626 return actionIf(LegalizeAction::Lower, Predicate, Mutation);
627 }
628 /// The instruction is lowered when type index 0 is any type in the given
629 /// list. Keep type index 0 as the same type.
lowerFor(std::initializer_list<LLT> Types)630 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) {
631 return actionFor(LegalizeAction::Lower, Types);
632 }
633 /// The instruction is lowered when type index 0 is any type in the given
634 /// list.
lowerFor(std::initializer_list<LLT> Types,LegalizeMutation Mutation)635 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types,
636 LegalizeMutation Mutation) {
637 return actionFor(LegalizeAction::Lower, Types, Mutation);
638 }
639 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
640 /// the given list. Keep type index 0 as the same type.
lowerFor(std::initializer_list<std::pair<LLT,LLT>> Types)641 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
642 return actionFor(LegalizeAction::Lower, Types);
643 }
644 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
645 /// the given list.
lowerFor(std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)646 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types,
647 LegalizeMutation Mutation) {
648 return actionFor(LegalizeAction::Lower, Types, Mutation);
649 }
650 /// The instruction is lowered when type indexes 0 and 1 are both in their
651 /// respective lists.
lowerForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)652 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
653 std::initializer_list<LLT> Types1) {
654 using namespace LegalityPredicates;
655 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1);
656 }
657 /// The instruction is lowered when when type indexes 0, 1, and 2 are all in
658 /// their respective lists.
lowerForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1,std::initializer_list<LLT> Types2)659 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
660 std::initializer_list<LLT> Types1,
661 std::initializer_list<LLT> Types2) {
662 using namespace LegalityPredicates;
663 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1,
664 Types2);
665 }
666
667 /// The instruction is emitted as a library call.
libcall()668 LegalizeRuleSet &libcall() {
669 using namespace LegalizeMutations;
670 // We have no choice but conservatively assume that predicate-less lowering
671 // properly handles all type indices by design:
672 markAllIdxsAsCovered();
673 return actionIf(LegalizeAction::Libcall, always);
674 }
675
676 /// Like legalIf, but for the Libcall action.
libcallIf(LegalityPredicate Predicate)677 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
678 // We have no choice but conservatively assume that a libcall with a
679 // free-form user provided Predicate properly handles all type indices:
680 markAllIdxsAsCovered();
681 return actionIf(LegalizeAction::Libcall, Predicate);
682 }
libcallFor(std::initializer_list<LLT> Types)683 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
684 return actionFor(LegalizeAction::Libcall, Types);
685 }
686 LegalizeRuleSet &
libcallFor(std::initializer_list<std::pair<LLT,LLT>> Types)687 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
688 return actionFor(LegalizeAction::Libcall, Types);
689 }
690 LegalizeRuleSet &
libcallForCartesianProduct(std::initializer_list<LLT> Types)691 libcallForCartesianProduct(std::initializer_list<LLT> Types) {
692 return actionForCartesianProduct(LegalizeAction::Libcall, Types);
693 }
694 LegalizeRuleSet &
libcallForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)695 libcallForCartesianProduct(std::initializer_list<LLT> Types0,
696 std::initializer_list<LLT> Types1) {
697 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1);
698 }
699
700 /// Widen the scalar to the one selected by the mutation if the predicate is
701 /// true.
widenScalarIf(LegalityPredicate Predicate,LegalizeMutation Mutation)702 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate,
703 LegalizeMutation Mutation) {
704 // We have no choice but conservatively assume that an action with a
705 // free-form user provided Predicate properly handles all type indices:
706 markAllIdxsAsCovered();
707 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation);
708 }
709 /// Narrow the scalar to the one selected by the mutation if the predicate is
710 /// true.
narrowScalarIf(LegalityPredicate Predicate,LegalizeMutation Mutation)711 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate,
712 LegalizeMutation Mutation) {
713 // We have no choice but conservatively assume that an action with a
714 // free-form user provided Predicate properly handles all type indices:
715 markAllIdxsAsCovered();
716 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation);
717 }
718 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any
719 /// type pair in the given list.
720 LegalizeRuleSet &
narrowScalarFor(std::initializer_list<std::pair<LLT,LLT>> Types,LegalizeMutation Mutation)721 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types,
722 LegalizeMutation Mutation) {
723 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation);
724 }
725
726 /// Add more elements to reach the type selected by the mutation if the
727 /// predicate is true.
moreElementsIf(LegalityPredicate Predicate,LegalizeMutation Mutation)728 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate,
729 LegalizeMutation Mutation) {
730 // We have no choice but conservatively assume that an action with a
731 // free-form user provided Predicate properly handles all type indices:
732 markAllIdxsAsCovered();
733 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation);
734 }
735 /// Remove elements to reach the type selected by the mutation if the
736 /// predicate is true.
fewerElementsIf(LegalityPredicate Predicate,LegalizeMutation Mutation)737 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate,
738 LegalizeMutation Mutation) {
739 // We have no choice but conservatively assume that an action with a
740 // free-form user provided Predicate properly handles all type indices:
741 markAllIdxsAsCovered();
742 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation);
743 }
744
745 /// The instruction is unsupported.
unsupported()746 LegalizeRuleSet &unsupported() {
747 markAllIdxsAsCovered();
748 return actionIf(LegalizeAction::Unsupported, always);
749 }
unsupportedIf(LegalityPredicate Predicate)750 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) {
751 return actionIf(LegalizeAction::Unsupported, Predicate);
752 }
753
unsupportedFor(std::initializer_list<LLT> Types)754 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) {
755 return actionFor(LegalizeAction::Unsupported, Types);
756 }
757
unsupportedIfMemSizeNotPow2()758 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() {
759 return actionIf(LegalizeAction::Unsupported,
760 LegalityPredicates::memSizeInBytesNotPow2(0));
761 }
lowerIfMemSizeNotPow2()762 LegalizeRuleSet &lowerIfMemSizeNotPow2() {
763 return actionIf(LegalizeAction::Lower,
764 LegalityPredicates::memSizeInBytesNotPow2(0));
765 }
766
customIf(LegalityPredicate Predicate)767 LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
768 // We have no choice but conservatively assume that a custom action with a
769 // free-form user provided Predicate properly handles all type indices:
770 markAllIdxsAsCovered();
771 return actionIf(LegalizeAction::Custom, Predicate);
772 }
customFor(std::initializer_list<LLT> Types)773 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
774 return actionFor(LegalizeAction::Custom, Types);
775 }
776
777 /// The instruction is custom when type indexes 0 and 1 is any type pair in the
778 /// given list.
customFor(std::initializer_list<std::pair<LLT,LLT>> Types)779 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
780 return actionFor(LegalizeAction::Custom, Types);
781 }
782
customForCartesianProduct(std::initializer_list<LLT> Types)783 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
784 return actionForCartesianProduct(LegalizeAction::Custom, Types);
785 }
786 LegalizeRuleSet &
customForCartesianProduct(std::initializer_list<LLT> Types0,std::initializer_list<LLT> Types1)787 customForCartesianProduct(std::initializer_list<LLT> Types0,
788 std::initializer_list<LLT> Types1) {
789 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1);
790 }
791
792 /// Unconditionally custom lower.
custom()793 LegalizeRuleSet &custom() {
794 return customIf(always);
795 }
796
797 /// Widen the scalar to the next power of two that is at least MinSize.
798 /// No effect if the type is not a scalar or is a power of two.
799 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx,
800 unsigned MinSize = 0) {
801 using namespace LegalityPredicates;
802 return actionIf(
803 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)),
804 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
805 }
806
807 /// Widen the scalar or vector element type to the next power of two that is
808 /// at least MinSize. No effect if the scalar size is a power of two.
809 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx,
810 unsigned MinSize = 0) {
811 using namespace LegalityPredicates;
812 return actionIf(
813 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)),
814 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
815 }
816
narrowScalar(unsigned TypeIdx,LegalizeMutation Mutation)817 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) {
818 using namespace LegalityPredicates;
819 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)),
820 Mutation);
821 }
822
scalarize(unsigned TypeIdx)823 LegalizeRuleSet &scalarize(unsigned TypeIdx) {
824 using namespace LegalityPredicates;
825 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)),
826 LegalizeMutations::scalarize(TypeIdx));
827 }
828
scalarizeIf(LegalityPredicate Predicate,unsigned TypeIdx)829 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) {
830 using namespace LegalityPredicates;
831 return actionIf(LegalizeAction::FewerElements,
832 all(Predicate, isVector(typeIdx(TypeIdx))),
833 LegalizeMutations::scalarize(TypeIdx));
834 }
835
836 /// Ensure the scalar or element is at least as wide as Ty.
minScalarOrElt(unsigned TypeIdx,const LLT Ty)837 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) {
838 using namespace LegalityPredicates;
839 using namespace LegalizeMutations;
840 return actionIf(LegalizeAction::WidenScalar,
841 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()),
842 changeElementTo(typeIdx(TypeIdx), Ty));
843 }
844
845 /// Ensure the scalar or element is at least as wide as Ty.
minScalarOrEltIf(LegalityPredicate Predicate,unsigned TypeIdx,const LLT Ty)846 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate,
847 unsigned TypeIdx, const LLT Ty) {
848 using namespace LegalityPredicates;
849 using namespace LegalizeMutations;
850 return actionIf(LegalizeAction::WidenScalar,
851 all(Predicate, scalarOrEltNarrowerThan(
852 TypeIdx, Ty.getScalarSizeInBits())),
853 changeElementTo(typeIdx(TypeIdx), Ty));
854 }
855
856 /// Ensure the scalar is at least as wide as Ty.
minScalar(unsigned TypeIdx,const LLT Ty)857 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) {
858 using namespace LegalityPredicates;
859 using namespace LegalizeMutations;
860 return actionIf(LegalizeAction::WidenScalar,
861 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()),
862 changeTo(typeIdx(TypeIdx), Ty));
863 }
864
865 /// Ensure the scalar is at most as wide as Ty.
maxScalarOrElt(unsigned TypeIdx,const LLT Ty)866 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) {
867 using namespace LegalityPredicates;
868 using namespace LegalizeMutations;
869 return actionIf(LegalizeAction::NarrowScalar,
870 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()),
871 changeElementTo(typeIdx(TypeIdx), Ty));
872 }
873
874 /// Ensure the scalar is at most as wide as Ty.
maxScalar(unsigned TypeIdx,const LLT Ty)875 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) {
876 using namespace LegalityPredicates;
877 using namespace LegalizeMutations;
878 return actionIf(LegalizeAction::NarrowScalar,
879 scalarWiderThan(TypeIdx, Ty.getSizeInBits()),
880 changeTo(typeIdx(TypeIdx), Ty));
881 }
882
883 /// Conditionally limit the maximum size of the scalar.
884 /// For example, when the maximum size of one type depends on the size of
885 /// another such as extracting N bits from an M bit container.
maxScalarIf(LegalityPredicate Predicate,unsigned TypeIdx,const LLT Ty)886 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
887 const LLT Ty) {
888 using namespace LegalityPredicates;
889 using namespace LegalizeMutations;
890 return actionIf(
891 LegalizeAction::NarrowScalar,
892 [=](const LegalityQuery &Query) {
893 const LLT QueryTy = Query.Types[TypeIdx];
894 return QueryTy.isScalar() &&
895 QueryTy.getSizeInBits() > Ty.getSizeInBits() &&
896 Predicate(Query);
897 },
898 changeElementTo(typeIdx(TypeIdx), Ty));
899 }
900
901 /// Limit the range of scalar sizes to MinTy and MaxTy.
clampScalar(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)902 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy,
903 const LLT MaxTy) {
904 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types");
905 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy);
906 }
907
908 /// Limit the range of scalar sizes to MinTy and MaxTy.
clampScalarOrElt(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)909 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy,
910 const LLT MaxTy) {
911 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy);
912 }
913
914 /// Widen the scalar to match the size of another.
minScalarSameAs(unsigned TypeIdx,unsigned LargeTypeIdx)915 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) {
916 typeIdx(TypeIdx);
917 return widenScalarIf(
918 [=](const LegalityQuery &Query) {
919 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
920 Query.Types[TypeIdx].getSizeInBits();
921 },
922 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx));
923 }
924
925 /// Narrow the scalar to match the size of another.
maxScalarSameAs(unsigned TypeIdx,unsigned NarrowTypeIdx)926 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) {
927 typeIdx(TypeIdx);
928 return narrowScalarIf(
929 [=](const LegalityQuery &Query) {
930 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() <
931 Query.Types[TypeIdx].getSizeInBits();
932 },
933 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx));
934 }
935
936 /// Change the type \p TypeIdx to have the same scalar size as type \p
937 /// SameSizeIdx.
scalarSameSizeAs(unsigned TypeIdx,unsigned SameSizeIdx)938 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) {
939 return minScalarSameAs(TypeIdx, SameSizeIdx)
940 .maxScalarSameAs(TypeIdx, SameSizeIdx);
941 }
942
943 /// Conditionally widen the scalar or elt to match the size of another.
minScalarEltSameAsIf(LegalityPredicate Predicate,unsigned TypeIdx,unsigned LargeTypeIdx)944 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate,
945 unsigned TypeIdx, unsigned LargeTypeIdx) {
946 typeIdx(TypeIdx);
947 return widenScalarIf(
948 [=](const LegalityQuery &Query) {
949 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
950 Query.Types[TypeIdx].getScalarSizeInBits() &&
951 Predicate(Query);
952 },
953 [=](const LegalityQuery &Query) {
954 LLT T = Query.Types[LargeTypeIdx];
955 return std::make_pair(TypeIdx, T);
956 });
957 }
958
959 /// Add more elements to the vector to reach the next power of two.
960 /// No effect if the type is not a vector or the element count is a power of
961 /// two.
moreElementsToNextPow2(unsigned TypeIdx)962 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) {
963 using namespace LegalityPredicates;
964 return actionIf(LegalizeAction::MoreElements,
965 numElementsNotPow2(typeIdx(TypeIdx)),
966 LegalizeMutations::moreElementsToNextPow2(TypeIdx));
967 }
968
969 /// Limit the number of elements in EltTy vectors to at least MinElements.
clampMinNumElements(unsigned TypeIdx,const LLT EltTy,unsigned MinElements)970 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy,
971 unsigned MinElements) {
972 // Mark the type index as covered:
973 typeIdx(TypeIdx);
974 return actionIf(
975 LegalizeAction::MoreElements,
976 [=](const LegalityQuery &Query) {
977 LLT VecTy = Query.Types[TypeIdx];
978 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
979 VecTy.getNumElements() < MinElements;
980 },
981 [=](const LegalityQuery &Query) {
982 LLT VecTy = Query.Types[TypeIdx];
983 return std::make_pair(
984 TypeIdx, LLT::vector(MinElements, VecTy.getElementType()));
985 });
986 }
987 /// Limit the number of elements in EltTy vectors to at most MaxElements.
clampMaxNumElements(unsigned TypeIdx,const LLT EltTy,unsigned MaxElements)988 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy,
989 unsigned MaxElements) {
990 // Mark the type index as covered:
991 typeIdx(TypeIdx);
992 return actionIf(
993 LegalizeAction::FewerElements,
994 [=](const LegalityQuery &Query) {
995 LLT VecTy = Query.Types[TypeIdx];
996 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
997 VecTy.getNumElements() > MaxElements;
998 },
999 [=](const LegalityQuery &Query) {
1000 LLT VecTy = Query.Types[TypeIdx];
1001 LLT NewTy = LLT::scalarOrVector(MaxElements, VecTy.getElementType());
1002 return std::make_pair(TypeIdx, NewTy);
1003 });
1004 }
1005 /// Limit the number of elements for the given vectors to at least MinTy's
1006 /// number of elements and at most MaxTy's number of elements.
1007 ///
1008 /// No effect if the type is not a vector or does not have the same element
1009 /// type as the constraints.
1010 /// The element type of MinTy and MaxTy must match.
clampNumElements(unsigned TypeIdx,const LLT MinTy,const LLT MaxTy)1011 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy,
1012 const LLT MaxTy) {
1013 assert(MinTy.getElementType() == MaxTy.getElementType() &&
1014 "Expected element types to agree");
1015
1016 const LLT EltTy = MinTy.getElementType();
1017 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements())
1018 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements());
1019 }
1020
1021 /// Fallback on the previous implementation. This should only be used while
1022 /// porting a rule.
fallback()1023 LegalizeRuleSet &fallback() {
1024 add({always, LegalizeAction::UseLegacyRules});
1025 return *this;
1026 }
1027
1028 /// Check if there is no type index which is obviously not handled by the
1029 /// LegalizeRuleSet in any way at all.
1030 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1031 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const;
1032 /// Check if there is no imm index which is obviously not handled by the
1033 /// LegalizeRuleSet in any way at all.
1034 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1035 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const;
1036
1037 /// Apply the ruleset to the given LegalityQuery.
1038 LegalizeActionStep apply(const LegalityQuery &Query) const;
1039 };
1040
1041 class LegalizerInfo {
1042 public:
1043 LegalizerInfo();
1044 virtual ~LegalizerInfo() = default;
1045
1046 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const;
1047 unsigned getActionDefinitionsIdx(unsigned Opcode) const;
1048
1049 /// Compute any ancillary tables needed to quickly decide how an operation
1050 /// should be handled. This must be called after all "set*Action"methods but
1051 /// before any query is made or incorrect results may be returned.
1052 void computeTables();
1053
1054 /// Perform simple self-diagnostic and assert if there is anything obviously
1055 /// wrong with the actions set up.
1056 void verify(const MCInstrInfo &MII) const;
1057
needsLegalizingToDifferentSize(const LegalizeAction Action)1058 static bool needsLegalizingToDifferentSize(const LegalizeAction Action) {
1059 using namespace LegalizeActions;
1060 switch (Action) {
1061 case NarrowScalar:
1062 case WidenScalar:
1063 case FewerElements:
1064 case MoreElements:
1065 case Unsupported:
1066 return true;
1067 default:
1068 return false;
1069 }
1070 }
1071
1072 using SizeAndAction = std::pair<uint16_t, LegalizeAction>;
1073 using SizeAndActionsVec = std::vector<SizeAndAction>;
1074 using SizeChangeStrategy =
1075 std::function<SizeAndActionsVec(const SizeAndActionsVec &v)>;
1076
1077 /// More friendly way to set an action for common types that have an LLT
1078 /// representation.
1079 /// The LegalizeAction must be one for which NeedsLegalizingToDifferentSize
1080 /// returns false.
setAction(const InstrAspect & Aspect,LegalizeAction Action)1081 void setAction(const InstrAspect &Aspect, LegalizeAction Action) {
1082 assert(!needsLegalizingToDifferentSize(Action));
1083 TablesInitialized = false;
1084 const unsigned OpcodeIdx = Aspect.Opcode - FirstOp;
1085 if (SpecifiedActions[OpcodeIdx].size() <= Aspect.Idx)
1086 SpecifiedActions[OpcodeIdx].resize(Aspect.Idx + 1);
1087 SpecifiedActions[OpcodeIdx][Aspect.Idx][Aspect.Type] = Action;
1088 }
1089
1090 /// The setAction calls record the non-size-changing legalization actions
1091 /// to take on specificly-sized types. The SizeChangeStrategy defines what
1092 /// to do when the size of the type needs to be changed to reach a legally
1093 /// sized type (i.e., one that was defined through a setAction call).
1094 /// e.g.
1095 /// setAction ({G_ADD, 0, LLT::scalar(32)}, Legal);
1096 /// setLegalizeScalarToDifferentSizeStrategy(
1097 /// G_ADD, 0, widenToLargerTypesAndNarrowToLargest);
1098 /// will end up defining getAction({G_ADD, 0, T}) to return the following
1099 /// actions for different scalar types T:
1100 /// LLT::scalar(1)..LLT::scalar(31): {WidenScalar, 0, LLT::scalar(32)}
1101 /// LLT::scalar(32): {Legal, 0, LLT::scalar(32)}
1102 /// LLT::scalar(33)..: {NarrowScalar, 0, LLT::scalar(32)}
1103 ///
1104 /// If no SizeChangeAction gets defined, through this function,
1105 /// the default is unsupportedForDifferentSizes.
setLegalizeScalarToDifferentSizeStrategy(const unsigned Opcode,const unsigned TypeIdx,SizeChangeStrategy S)1106 void setLegalizeScalarToDifferentSizeStrategy(const unsigned Opcode,
1107 const unsigned TypeIdx,
1108 SizeChangeStrategy S) {
1109 const unsigned OpcodeIdx = Opcode - FirstOp;
1110 if (ScalarSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx)
1111 ScalarSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1);
1112 ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] = S;
1113 }
1114
1115 /// See also setLegalizeScalarToDifferentSizeStrategy.
1116 /// This function allows to set the SizeChangeStrategy for vector elements.
setLegalizeVectorElementToDifferentSizeStrategy(const unsigned Opcode,const unsigned TypeIdx,SizeChangeStrategy S)1117 void setLegalizeVectorElementToDifferentSizeStrategy(const unsigned Opcode,
1118 const unsigned TypeIdx,
1119 SizeChangeStrategy S) {
1120 const unsigned OpcodeIdx = Opcode - FirstOp;
1121 if (VectorElementSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx)
1122 VectorElementSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1);
1123 VectorElementSizeChangeStrategies[OpcodeIdx][TypeIdx] = S;
1124 }
1125
1126 /// A SizeChangeStrategy for the common case where legalization for a
1127 /// particular operation consists of only supporting a specific set of type
1128 /// sizes. E.g.
1129 /// setAction ({G_DIV, 0, LLT::scalar(32)}, Legal);
1130 /// setAction ({G_DIV, 0, LLT::scalar(64)}, Legal);
1131 /// setLegalizeScalarToDifferentSizeStrategy(
1132 /// G_DIV, 0, unsupportedForDifferentSizes);
1133 /// will result in getAction({G_DIV, 0, T}) to return Legal for s32 and s64,
1134 /// and Unsupported for all other scalar types T.
1135 static SizeAndActionsVec
unsupportedForDifferentSizes(const SizeAndActionsVec & v)1136 unsupportedForDifferentSizes(const SizeAndActionsVec &v) {
1137 using namespace LegalizeActions;
1138 return increaseToLargerTypesAndDecreaseToLargest(v, Unsupported,
1139 Unsupported);
1140 }
1141
1142 /// A SizeChangeStrategy for the common case where legalization for a
1143 /// particular operation consists of widening the type to a large legal type,
1144 /// unless there is no such type and then instead it should be narrowed to the
1145 /// largest legal type.
1146 static SizeAndActionsVec
widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec & v)1147 widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) {
1148 using namespace LegalizeActions;
1149 assert(v.size() > 0 &&
1150 "At least one size that can be legalized towards is needed"
1151 " for this SizeChangeStrategy");
1152 return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar,
1153 NarrowScalar);
1154 }
1155
1156 static SizeAndActionsVec
widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec & v)1157 widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec &v) {
1158 using namespace LegalizeActions;
1159 return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar,
1160 Unsupported);
1161 }
1162
1163 static SizeAndActionsVec
narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec & v)1164 narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec &v) {
1165 using namespace LegalizeActions;
1166 return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar,
1167 Unsupported);
1168 }
1169
1170 static SizeAndActionsVec
narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec & v)1171 narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) {
1172 using namespace LegalizeActions;
1173 assert(v.size() > 0 &&
1174 "At least one size that can be legalized towards is needed"
1175 " for this SizeChangeStrategy");
1176 return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar,
1177 WidenScalar);
1178 }
1179
1180 /// A SizeChangeStrategy for the common case where legalization for a
1181 /// particular vector operation consists of having more elements in the
1182 /// vector, to a type that is legal. Unless there is no such type and then
1183 /// instead it should be legalized towards the widest vector that's still
1184 /// legal. E.g.
1185 /// setAction({G_ADD, LLT::vector(8, 8)}, Legal);
1186 /// setAction({G_ADD, LLT::vector(16, 8)}, Legal);
1187 /// setAction({G_ADD, LLT::vector(2, 32)}, Legal);
1188 /// setAction({G_ADD, LLT::vector(4, 32)}, Legal);
1189 /// setLegalizeVectorElementToDifferentSizeStrategy(
1190 /// G_ADD, 0, moreToWiderTypesAndLessToWidest);
1191 /// will result in the following getAction results:
1192 /// * getAction({G_ADD, LLT::vector(8,8)}) returns
1193 /// (Legal, vector(8,8)).
1194 /// * getAction({G_ADD, LLT::vector(9,8)}) returns
1195 /// (MoreElements, vector(16,8)).
1196 /// * getAction({G_ADD, LLT::vector(8,32)}) returns
1197 /// (FewerElements, vector(4,32)).
1198 static SizeAndActionsVec
moreToWiderTypesAndLessToWidest(const SizeAndActionsVec & v)1199 moreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) {
1200 using namespace LegalizeActions;
1201 return increaseToLargerTypesAndDecreaseToLargest(v, MoreElements,
1202 FewerElements);
1203 }
1204
1205 /// Helper function to implement many typical SizeChangeStrategy functions.
1206 static SizeAndActionsVec
1207 increaseToLargerTypesAndDecreaseToLargest(const SizeAndActionsVec &v,
1208 LegalizeAction IncreaseAction,
1209 LegalizeAction DecreaseAction);
1210 /// Helper function to implement many typical SizeChangeStrategy functions.
1211 static SizeAndActionsVec
1212 decreaseToSmallerTypesAndIncreaseToSmallest(const SizeAndActionsVec &v,
1213 LegalizeAction DecreaseAction,
1214 LegalizeAction IncreaseAction);
1215
1216 /// Get the action definitions for the given opcode. Use this to run a
1217 /// LegalityQuery through the definitions.
1218 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const;
1219
1220 /// Get the action definition builder for the given opcode. Use this to define
1221 /// the action definitions.
1222 ///
1223 /// It is an error to request an opcode that has already been requested by the
1224 /// multiple-opcode variant.
1225 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode);
1226
1227 /// Get the action definition builder for the given set of opcodes. Use this
1228 /// to define the action definitions for multiple opcodes at once. The first
1229 /// opcode given will be considered the representative opcode and will hold
1230 /// the definitions whereas the other opcodes will be configured to refer to
1231 /// the representative opcode. This lowers memory requirements and very
1232 /// slightly improves performance.
1233 ///
1234 /// It would be very easy to introduce unexpected side-effects as a result of
1235 /// this aliasing if it were permitted to request different but intersecting
1236 /// sets of opcodes but that is difficult to keep track of. It is therefore an
1237 /// error to request the same opcode twice using this API, to request an
1238 /// opcode that already has definitions, or to use the single-opcode API on an
1239 /// opcode that has already been requested by this API.
1240 LegalizeRuleSet &
1241 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes);
1242 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom);
1243
1244 /// Determine what action should be taken to legalize the described
1245 /// instruction. Requires computeTables to have been called.
1246 ///
1247 /// \returns a description of the next legalization step to perform.
1248 LegalizeActionStep getAction(const LegalityQuery &Query) const;
1249
1250 /// Determine what action should be taken to legalize the given generic
1251 /// instruction.
1252 ///
1253 /// \returns a description of the next legalization step to perform.
1254 LegalizeActionStep getAction(const MachineInstr &MI,
1255 const MachineRegisterInfo &MRI) const;
1256
isLegal(const LegalityQuery & Query)1257 bool isLegal(const LegalityQuery &Query) const {
1258 return getAction(Query).Action == LegalizeAction::Legal;
1259 }
1260
isLegalOrCustom(const LegalityQuery & Query)1261 bool isLegalOrCustom(const LegalityQuery &Query) const {
1262 auto Action = getAction(Query).Action;
1263 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom;
1264 }
1265
1266 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
1267 bool isLegalOrCustom(const MachineInstr &MI,
1268 const MachineRegisterInfo &MRI) const;
1269
1270 /// Called for instructions with the Custom LegalizationAction.
legalizeCustom(LegalizerHelper & Helper,MachineInstr & MI)1271 virtual bool legalizeCustom(LegalizerHelper &Helper,
1272 MachineInstr &MI) const {
1273 llvm_unreachable("must implement this if custom action is used");
1274 }
1275
1276 /// \returns true if MI is either legal or has been legalized and false if not
1277 /// legal.
1278 /// Return true if MI is either legal or has been legalized and false
1279 /// if not legal.
legalizeIntrinsic(LegalizerHelper & Helper,MachineInstr & MI)1280 virtual bool legalizeIntrinsic(LegalizerHelper &Helper,
1281 MachineInstr &MI) const {
1282 return true;
1283 }
1284
1285 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while
1286 /// widening a constant of type SmallTy which targets can override.
1287 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which
1288 /// will be the default.
1289 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const;
1290
1291 private:
1292 /// Determine what action should be taken to legalize the given generic
1293 /// instruction opcode, type-index and type. Requires computeTables to have
1294 /// been called.
1295 ///
1296 /// \returns a pair consisting of the kind of legalization that should be
1297 /// performed and the destination type.
1298 std::pair<LegalizeAction, LLT>
1299 getAspectAction(const InstrAspect &Aspect) const;
1300
1301 /// The SizeAndActionsVec is a representation mapping between all natural
1302 /// numbers and an Action. The natural number represents the bit size of
1303 /// the InstrAspect. For example, for a target with native support for 32-bit
1304 /// and 64-bit additions, you'd express that as:
1305 /// setScalarAction(G_ADD, 0,
1306 /// {{1, WidenScalar}, // bit sizes [ 1, 31[
1307 /// {32, Legal}, // bit sizes [32, 33[
1308 /// {33, WidenScalar}, // bit sizes [33, 64[
1309 /// {64, Legal}, // bit sizes [64, 65[
1310 /// {65, NarrowScalar} // bit sizes [65, +inf[
1311 /// });
1312 /// It may be that only 64-bit pointers are supported on your target:
1313 /// setPointerAction(G_PTR_ADD, 0, LLT:pointer(1),
1314 /// {{1, Unsupported}, // bit sizes [ 1, 63[
1315 /// {64, Legal}, // bit sizes [64, 65[
1316 /// {65, Unsupported}, // bit sizes [65, +inf[
1317 /// });
setScalarAction(const unsigned Opcode,const unsigned TypeIndex,const SizeAndActionsVec & SizeAndActions)1318 void setScalarAction(const unsigned Opcode, const unsigned TypeIndex,
1319 const SizeAndActionsVec &SizeAndActions) {
1320 const unsigned OpcodeIdx = Opcode - FirstOp;
1321 SmallVector<SizeAndActionsVec, 1> &Actions = ScalarActions[OpcodeIdx];
1322 setActions(TypeIndex, Actions, SizeAndActions);
1323 }
setPointerAction(const unsigned Opcode,const unsigned TypeIndex,const unsigned AddressSpace,const SizeAndActionsVec & SizeAndActions)1324 void setPointerAction(const unsigned Opcode, const unsigned TypeIndex,
1325 const unsigned AddressSpace,
1326 const SizeAndActionsVec &SizeAndActions) {
1327 const unsigned OpcodeIdx = Opcode - FirstOp;
1328 if (AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace) ==
1329 AddrSpace2PointerActions[OpcodeIdx].end())
1330 AddrSpace2PointerActions[OpcodeIdx][AddressSpace] = {{}};
1331 SmallVector<SizeAndActionsVec, 1> &Actions =
1332 AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace)->second;
1333 setActions(TypeIndex, Actions, SizeAndActions);
1334 }
1335
1336 /// If an operation on a given vector type (say <M x iN>) isn't explicitly
1337 /// specified, we proceed in 2 stages. First we legalize the underlying scalar
1338 /// (so that there's at least one legal vector with that scalar), then we
1339 /// adjust the number of elements in the vector so that it is legal. The
1340 /// desired action in the first step is controlled by this function.
setScalarInVectorAction(const unsigned Opcode,const unsigned TypeIndex,const SizeAndActionsVec & SizeAndActions)1341 void setScalarInVectorAction(const unsigned Opcode, const unsigned TypeIndex,
1342 const SizeAndActionsVec &SizeAndActions) {
1343 unsigned OpcodeIdx = Opcode - FirstOp;
1344 SmallVector<SizeAndActionsVec, 1> &Actions =
1345 ScalarInVectorActions[OpcodeIdx];
1346 setActions(TypeIndex, Actions, SizeAndActions);
1347 }
1348
1349 /// See also setScalarInVectorAction.
1350 /// This function let's you specify the number of elements in a vector that
1351 /// are legal for a legal element size.
setVectorNumElementAction(const unsigned Opcode,const unsigned TypeIndex,const unsigned ElementSize,const SizeAndActionsVec & SizeAndActions)1352 void setVectorNumElementAction(const unsigned Opcode,
1353 const unsigned TypeIndex,
1354 const unsigned ElementSize,
1355 const SizeAndActionsVec &SizeAndActions) {
1356 const unsigned OpcodeIdx = Opcode - FirstOp;
1357 if (NumElements2Actions[OpcodeIdx].find(ElementSize) ==
1358 NumElements2Actions[OpcodeIdx].end())
1359 NumElements2Actions[OpcodeIdx][ElementSize] = {{}};
1360 SmallVector<SizeAndActionsVec, 1> &Actions =
1361 NumElements2Actions[OpcodeIdx].find(ElementSize)->second;
1362 setActions(TypeIndex, Actions, SizeAndActions);
1363 }
1364
1365 /// A partial SizeAndActionsVec potentially doesn't cover all bit sizes,
1366 /// i.e. it's OK if it doesn't start from size 1.
checkPartialSizeAndActionsVector(const SizeAndActionsVec & v)1367 static void checkPartialSizeAndActionsVector(const SizeAndActionsVec& v) {
1368 using namespace LegalizeActions;
1369 #ifndef NDEBUG
1370 // The sizes should be in increasing order
1371 int prev_size = -1;
1372 for(auto SizeAndAction: v) {
1373 assert(SizeAndAction.first > prev_size);
1374 prev_size = SizeAndAction.first;
1375 }
1376 // - for every Widen action, there should be a larger bitsize that
1377 // can be legalized towards (e.g. Legal, Lower, Libcall or Custom
1378 // action).
1379 // - for every Narrow action, there should be a smaller bitsize that
1380 // can be legalized towards.
1381 int SmallestNarrowIdx = -1;
1382 int LargestWidenIdx = -1;
1383 int SmallestLegalizableToSameSizeIdx = -1;
1384 int LargestLegalizableToSameSizeIdx = -1;
1385 for(size_t i=0; i<v.size(); ++i) {
1386 switch (v[i].second) {
1387 case FewerElements:
1388 case NarrowScalar:
1389 if (SmallestNarrowIdx == -1)
1390 SmallestNarrowIdx = i;
1391 break;
1392 case WidenScalar:
1393 case MoreElements:
1394 LargestWidenIdx = i;
1395 break;
1396 case Unsupported:
1397 break;
1398 default:
1399 if (SmallestLegalizableToSameSizeIdx == -1)
1400 SmallestLegalizableToSameSizeIdx = i;
1401 LargestLegalizableToSameSizeIdx = i;
1402 }
1403 }
1404 if (SmallestNarrowIdx != -1) {
1405 assert(SmallestLegalizableToSameSizeIdx != -1);
1406 assert(SmallestNarrowIdx > SmallestLegalizableToSameSizeIdx);
1407 }
1408 if (LargestWidenIdx != -1)
1409 assert(LargestWidenIdx < LargestLegalizableToSameSizeIdx);
1410 #endif
1411 }
1412
1413 /// A full SizeAndActionsVec must cover all bit sizes, i.e. must start with
1414 /// from size 1.
checkFullSizeAndActionsVector(const SizeAndActionsVec & v)1415 static void checkFullSizeAndActionsVector(const SizeAndActionsVec& v) {
1416 #ifndef NDEBUG
1417 // Data structure invariant: The first bit size must be size 1.
1418 assert(v.size() >= 1);
1419 assert(v[0].first == 1);
1420 checkPartialSizeAndActionsVector(v);
1421 #endif
1422 }
1423
1424 /// Sets actions for all bit sizes on a particular generic opcode, type
1425 /// index and scalar or pointer type.
setActions(unsigned TypeIndex,SmallVector<SizeAndActionsVec,1> & Actions,const SizeAndActionsVec & SizeAndActions)1426 void setActions(unsigned TypeIndex,
1427 SmallVector<SizeAndActionsVec, 1> &Actions,
1428 const SizeAndActionsVec &SizeAndActions) {
1429 checkFullSizeAndActionsVector(SizeAndActions);
1430 if (Actions.size() <= TypeIndex)
1431 Actions.resize(TypeIndex + 1);
1432 Actions[TypeIndex] = SizeAndActions;
1433 }
1434
1435 static SizeAndAction findAction(const SizeAndActionsVec &Vec,
1436 const uint32_t Size);
1437
1438 /// Returns the next action needed to get the scalar or pointer type closer
1439 /// to being legal
1440 /// E.g. findLegalAction({G_REM, 13}) should return
1441 /// (WidenScalar, 32). After that, findLegalAction({G_REM, 32}) will
1442 /// probably be called, which should return (Lower, 32).
1443 /// This is assuming the setScalarAction on G_REM was something like:
1444 /// setScalarAction(G_REM, 0,
1445 /// {{1, WidenScalar}, // bit sizes [ 1, 31[
1446 /// {32, Lower}, // bit sizes [32, 33[
1447 /// {33, NarrowScalar} // bit sizes [65, +inf[
1448 /// });
1449 std::pair<LegalizeAction, LLT>
1450 findScalarLegalAction(const InstrAspect &Aspect) const;
1451
1452 /// Returns the next action needed towards legalizing the vector type.
1453 std::pair<LegalizeAction, LLT>
1454 findVectorLegalAction(const InstrAspect &Aspect) const;
1455
1456 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START;
1457 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END;
1458
1459 // Data structures used temporarily during construction of legality data:
1460 using TypeMap = DenseMap<LLT, LegalizeAction>;
1461 SmallVector<TypeMap, 1> SpecifiedActions[LastOp - FirstOp + 1];
1462 SmallVector<SizeChangeStrategy, 1>
1463 ScalarSizeChangeStrategies[LastOp - FirstOp + 1];
1464 SmallVector<SizeChangeStrategy, 1>
1465 VectorElementSizeChangeStrategies[LastOp - FirstOp + 1];
1466 bool TablesInitialized;
1467
1468 // Data structures used by getAction:
1469 SmallVector<SizeAndActionsVec, 1> ScalarActions[LastOp - FirstOp + 1];
1470 SmallVector<SizeAndActionsVec, 1> ScalarInVectorActions[LastOp - FirstOp + 1];
1471 std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>>
1472 AddrSpace2PointerActions[LastOp - FirstOp + 1];
1473 std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>>
1474 NumElements2Actions[LastOp - FirstOp + 1];
1475
1476 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1];
1477 };
1478
1479 #ifndef NDEBUG
1480 /// Checks that MIR is fully legal, returns an illegal instruction if it's not,
1481 /// nullptr otherwise
1482 const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF);
1483 #endif
1484
1485 } // end namespace llvm.
1486
1487 #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
1488