1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/Utils.h"
21 #include "llvm/CodeGen/MachineInstrBuilder.h"
22 #include "llvm/CodeGen/MachineOperand.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/CodeGen/RegisterBankInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/CodeGenCoverage.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/ErrorHandling.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include <cassert>
35 #include <cstddef>
36 #include <cstdint>
37 
38 namespace llvm {
39 
40 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
41           class CustomRendererFn>
42 bool GIMatchTableExecutor::executeMatchTable(
43     TgtExecutor &Exec, NewMIVector &OutMIs, MatcherState &State,
44     const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
45         &ExecInfo,
46     const int64_t *MatchTable, const TargetInstrInfo &TII,
47     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
48     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
49     CodeGenCoverage *CoverageInfo) const {
50 
51   uint64_t CurrentIdx = 0;
52   SmallVector<uint64_t, 4> OnFailResumeAt;
53 
54   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
55   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
56 
57   const uint16_t Flags = State.MIs[0]->getFlags();
58 
59   enum RejectAction { RejectAndGiveUp, RejectAndResume };
60   auto handleReject = [&]() -> RejectAction {
61     DEBUG_WITH_TYPE(TgtExecutor::getName(),
62                     dbgs() << CurrentIdx << ": Rejected\n");
63     if (OnFailResumeAt.empty())
64       return RejectAndGiveUp;
65     CurrentIdx = OnFailResumeAt.pop_back_val();
66     DEBUG_WITH_TYPE(TgtExecutor::getName(),
67                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
68                            << OnFailResumeAt.size() << " try-blocks remain)\n");
69     return RejectAndResume;
70   };
71 
72   auto propagateFlags = [=](NewMIVector &OutMIs) {
73     for (auto MIB : OutMIs) {
74       // Set the NoFPExcept flag when no original matched instruction could
75       // raise an FP exception, but the new instruction potentially might.
76       uint16_t MIBFlags = Flags;
77       if (NoFPException && MIB->mayRaiseFPException())
78         MIBFlags |= MachineInstr::NoFPExcept;
79       MIB.setMIFlags(MIBFlags);
80     }
81 
82     return true;
83   };
84 
85   while (true) {
86     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
87     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
88     switch (MatcherOpcode) {
89     case GIM_Try: {
90       DEBUG_WITH_TYPE(TgtExecutor::getName(),
91                       dbgs() << CurrentIdx << ": Begin try-block\n");
92       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
93       break;
94     }
95 
96     case GIM_RecordInsn:
97     case GIM_RecordInsnIgnoreCopies: {
98       int64_t NewInsnID = MatchTable[CurrentIdx++];
99       int64_t InsnID = MatchTable[CurrentIdx++];
100       int64_t OpIdx = MatchTable[CurrentIdx++];
101 
102       // As an optimisation we require that MIs[0] is always the root. Refuse
103       // any attempt to modify it.
104       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
105 
106       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
107       if (!MO.isReg()) {
108         DEBUG_WITH_TYPE(TgtExecutor::getName(),
109                         dbgs() << CurrentIdx << ": Not a register\n");
110         if (handleReject() == RejectAndGiveUp)
111           return false;
112         break;
113       }
114       if (MO.getReg().isPhysical()) {
115         DEBUG_WITH_TYPE(TgtExecutor::getName(),
116                         dbgs() << CurrentIdx << ": Is a physical register\n");
117         if (handleReject() == RejectAndGiveUp)
118           return false;
119         break;
120       }
121 
122       MachineInstr *NewMI;
123       if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
124         NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
125       else
126         NewMI = MRI.getVRegDef(MO.getReg());
127 
128       if ((size_t)NewInsnID < State.MIs.size())
129         State.MIs[NewInsnID] = NewMI;
130       else {
131         assert((size_t)NewInsnID == State.MIs.size() &&
132                "Expected to store MIs in order");
133         State.MIs.push_back(NewMI);
134       }
135       DEBUG_WITH_TYPE(TgtExecutor::getName(),
136                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
137                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
138                              << ")\n");
139       break;
140     }
141 
142     case GIM_CheckFeatures: {
143       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
144       DEBUG_WITH_TYPE(TgtExecutor::getName(),
145                       dbgs() << CurrentIdx
146                              << ": GIM_CheckFeatures(ExpectedBitsetID="
147                              << ExpectedBitsetID << ")\n");
148       if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
149           ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
150         if (handleReject() == RejectAndGiveUp)
151           return false;
152       }
153       break;
154     }
155     case GIM_CheckOpcode:
156     case GIM_CheckOpcodeIsEither: {
157       int64_t InsnID = MatchTable[CurrentIdx++];
158       int64_t Expected0 = MatchTable[CurrentIdx++];
159       int64_t Expected1 = -1;
160       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
161         Expected1 = MatchTable[CurrentIdx++];
162 
163       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164       unsigned Opcode = State.MIs[InsnID]->getOpcode();
165 
166       DEBUG_WITH_TYPE(TgtExecutor::getName(),
167                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
168                              << "], ExpectedOpcode=" << Expected0;
169                       if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
170                       << " || " << Expected1;
171                       dbgs() << ") // Got=" << Opcode << "\n";);
172 
173       if (Opcode != Expected0 && Opcode != Expected1) {
174         if (handleReject() == RejectAndGiveUp)
175           return false;
176       }
177       break;
178     }
179     case GIM_SwitchOpcode: {
180       int64_t InsnID = MatchTable[CurrentIdx++];
181       int64_t LowerBound = MatchTable[CurrentIdx++];
182       int64_t UpperBound = MatchTable[CurrentIdx++];
183       int64_t Default = MatchTable[CurrentIdx++];
184 
185       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
186       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
187 
188       DEBUG_WITH_TYPE(TgtExecutor::getName(), {
189         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
190                << LowerBound << ", " << UpperBound << "), Default=" << Default
191                << ", JumpTable...) // Got=" << Opcode << "\n";
192       });
193       if (Opcode < LowerBound || UpperBound <= Opcode) {
194         CurrentIdx = Default;
195         break;
196       }
197       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
198       if (!CurrentIdx) {
199         CurrentIdx = Default;
200         break;
201       }
202       OnFailResumeAt.push_back(Default);
203       break;
204     }
205 
206     case GIM_SwitchType: {
207       int64_t InsnID = MatchTable[CurrentIdx++];
208       int64_t OpIdx = MatchTable[CurrentIdx++];
209       int64_t LowerBound = MatchTable[CurrentIdx++];
210       int64_t UpperBound = MatchTable[CurrentIdx++];
211       int64_t Default = MatchTable[CurrentIdx++];
212 
213       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
214       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
215 
216       DEBUG_WITH_TYPE(TgtExecutor::getName(), {
217         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
218                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
219                << UpperBound << "), Default=" << Default
220                << ", JumpTable...) // Got=";
221         if (!MO.isReg())
222           dbgs() << "Not a VReg\n";
223         else
224           dbgs() << MRI.getType(MO.getReg()) << "\n";
225       });
226       if (!MO.isReg()) {
227         CurrentIdx = Default;
228         break;
229       }
230       const LLT Ty = MRI.getType(MO.getReg());
231       const auto TyI = ExecInfo.TypeIDMap.find(Ty);
232       if (TyI == ExecInfo.TypeIDMap.end()) {
233         CurrentIdx = Default;
234         break;
235       }
236       const int64_t TypeID = TyI->second;
237       if (TypeID < LowerBound || UpperBound <= TypeID) {
238         CurrentIdx = Default;
239         break;
240       }
241       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
242       if (!CurrentIdx) {
243         CurrentIdx = Default;
244         break;
245       }
246       OnFailResumeAt.push_back(Default);
247       break;
248     }
249 
250     case GIM_CheckNumOperands: {
251       int64_t InsnID = MatchTable[CurrentIdx++];
252       int64_t Expected = MatchTable[CurrentIdx++];
253       DEBUG_WITH_TYPE(TgtExecutor::getName(),
254                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
255                              << InsnID << "], Expected=" << Expected << ")\n");
256       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
257       if (State.MIs[InsnID]->getNumOperands() != Expected) {
258         if (handleReject() == RejectAndGiveUp)
259           return false;
260       }
261       break;
262     }
263     case GIM_CheckI64ImmPredicate:
264     case GIM_CheckImmOperandPredicate: {
265       int64_t InsnID = MatchTable[CurrentIdx++];
266       int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
267                           ? MatchTable[CurrentIdx++]
268                           : 1;
269       int64_t Predicate = MatchTable[CurrentIdx++];
270       DEBUG_WITH_TYPE(TgtExecutor::getName(),
271                       dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
272                              << InsnID << "]->getOperand(" << OpIdx
273                              << "), Predicate=" << Predicate << ")\n");
274       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
275       assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
276               State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
277              "Expected immediate operand");
278       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
279       int64_t Value = 0;
280       if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
281         Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
282       else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
283         Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
284       else
285         llvm_unreachable("Expected Imm or CImm operand");
286 
287       if (!testImmPredicate_I64(Predicate, Value))
288         if (handleReject() == RejectAndGiveUp)
289           return false;
290       break;
291     }
292     case GIM_CheckAPIntImmPredicate: {
293       int64_t InsnID = MatchTable[CurrentIdx++];
294       int64_t Predicate = MatchTable[CurrentIdx++];
295       DEBUG_WITH_TYPE(TgtExecutor::getName(),
296                       dbgs()
297                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
298                           << InsnID << "], Predicate=" << Predicate << ")\n");
299       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
300       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
301              "Expected G_CONSTANT");
302       assert(Predicate > GICXXPred_Invalid &&
303              "Expected a valid predicate");
304       if (!State.MIs[InsnID]->getOperand(1).isCImm())
305         llvm_unreachable("Expected Imm or CImm operand");
306 
307       const APInt &Value =
308           State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
309       if (!testImmPredicate_APInt(Predicate, Value))
310         if (handleReject() == RejectAndGiveUp)
311           return false;
312       break;
313     }
314     case GIM_CheckAPFloatImmPredicate: {
315       int64_t InsnID = MatchTable[CurrentIdx++];
316       int64_t Predicate = MatchTable[CurrentIdx++];
317       DEBUG_WITH_TYPE(TgtExecutor::getName(),
318                       dbgs()
319                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
320                           << InsnID << "], Predicate=" << Predicate << ")\n");
321       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
322       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
323              "Expected G_FCONSTANT");
324       assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
325              "Expected FPImm operand");
326       assert(Predicate > GICXXPred_Invalid &&
327              "Expected a valid predicate");
328       const APFloat &Value =
329           State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
330 
331       if (!testImmPredicate_APFloat(Predicate, Value))
332         if (handleReject() == RejectAndGiveUp)
333           return false;
334       break;
335     }
336     case GIM_CheckIsBuildVectorAllOnes:
337     case GIM_CheckIsBuildVectorAllZeros: {
338       int64_t InsnID = MatchTable[CurrentIdx++];
339 
340       DEBUG_WITH_TYPE(TgtExecutor::getName(),
341                       dbgs() << CurrentIdx
342                              << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
343                              << InsnID << "])\n");
344       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345 
346       const MachineInstr *MI = State.MIs[InsnID];
347       assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
348               MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
349              "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
350 
351       if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
352         if (!isBuildVectorAllOnes(*MI, MRI)) {
353           if (handleReject() == RejectAndGiveUp)
354             return false;
355         }
356       } else {
357         if (!isBuildVectorAllZeros(*MI, MRI)) {
358           if (handleReject() == RejectAndGiveUp)
359             return false;
360         }
361       }
362 
363       break;
364     }
365     case GIM_CheckSimplePredicate: {
366       // Note: we don't check for invalid here because this is purely a hook to
367       // allow some executors (such as the combiner) to check arbitrary,
368       // contextless predicates, such as whether a rule is enabled or not.
369       int64_t Predicate = MatchTable[CurrentIdx++];
370       DEBUG_WITH_TYPE(TgtExecutor::getName(),
371                       dbgs() << CurrentIdx
372                              << ": GIM_CheckSimplePredicate(Predicate="
373                              << Predicate << ")\n");
374       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
375       if (!testSimplePredicate(Predicate)) {
376         if (handleReject() == RejectAndGiveUp)
377           return false;
378       }
379       break;
380     }
381     case GIM_CheckCxxInsnPredicate: {
382       int64_t InsnID = MatchTable[CurrentIdx++];
383       int64_t Predicate = MatchTable[CurrentIdx++];
384       DEBUG_WITH_TYPE(TgtExecutor::getName(),
385                       dbgs()
386                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
387                           << InsnID << "], Predicate=" << Predicate << ")\n");
388       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
389       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
390 
391       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
392         if (handleReject() == RejectAndGiveUp)
393           return false;
394       break;
395     }
396     case GIM_CheckHasNoUse: {
397       int64_t InsnID = MatchTable[CurrentIdx++];
398 
399       DEBUG_WITH_TYPE(TgtExecutor::getName(),
400                       dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
401                              << InsnID << "]\n");
402 
403       const MachineInstr *MI = State.MIs[InsnID];
404       assert(MI && "Used insn before defined");
405       assert(MI->getNumDefs() > 0 && "No defs");
406       const Register Res = MI->getOperand(0).getReg();
407 
408       if (!MRI.use_nodbg_empty(Res)) {
409         if (handleReject() == RejectAndGiveUp)
410           return false;
411       }
412 
413       break;
414     }
415     case GIM_CheckAtomicOrdering: {
416       int64_t InsnID = MatchTable[CurrentIdx++];
417       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
418       DEBUG_WITH_TYPE(TgtExecutor::getName(),
419                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
420                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
421       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
422       if (!State.MIs[InsnID]->hasOneMemOperand())
423         if (handleReject() == RejectAndGiveUp)
424           return false;
425 
426       for (const auto &MMO : State.MIs[InsnID]->memoperands())
427         if (MMO->getMergedOrdering() != Ordering)
428           if (handleReject() == RejectAndGiveUp)
429             return false;
430       break;
431     }
432     case GIM_CheckAtomicOrderingOrStrongerThan: {
433       int64_t InsnID = MatchTable[CurrentIdx++];
434       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
435       DEBUG_WITH_TYPE(TgtExecutor::getName(),
436                       dbgs() << CurrentIdx
437                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
438                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
439       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
440       if (!State.MIs[InsnID]->hasOneMemOperand())
441         if (handleReject() == RejectAndGiveUp)
442           return false;
443 
444       for (const auto &MMO : State.MIs[InsnID]->memoperands())
445         if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
446           if (handleReject() == RejectAndGiveUp)
447             return false;
448       break;
449     }
450     case GIM_CheckAtomicOrderingWeakerThan: {
451       int64_t InsnID = MatchTable[CurrentIdx++];
452       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
453       DEBUG_WITH_TYPE(TgtExecutor::getName(),
454                       dbgs() << CurrentIdx
455                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
456                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
457       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
458       if (!State.MIs[InsnID]->hasOneMemOperand())
459         if (handleReject() == RejectAndGiveUp)
460           return false;
461 
462       for (const auto &MMO : State.MIs[InsnID]->memoperands())
463         if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
464           if (handleReject() == RejectAndGiveUp)
465             return false;
466       break;
467     }
468     case GIM_CheckMemoryAddressSpace: {
469       int64_t InsnID = MatchTable[CurrentIdx++];
470       int64_t MMOIdx = MatchTable[CurrentIdx++];
471       // This accepts a list of possible address spaces.
472       const int NumAddrSpace = MatchTable[CurrentIdx++];
473 
474       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
475         if (handleReject() == RejectAndGiveUp)
476           return false;
477         break;
478       }
479 
480       // Need to still jump to the end of the list of address spaces if we find
481       // a match earlier.
482       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
483 
484       const MachineMemOperand *MMO =
485           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
486       const unsigned MMOAddrSpace = MMO->getAddrSpace();
487 
488       bool Success = false;
489       for (int I = 0; I != NumAddrSpace; ++I) {
490         unsigned AddrSpace = MatchTable[CurrentIdx++];
491         DEBUG_WITH_TYPE(TgtExecutor::getName(),
492                         dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
493                                << AddrSpace << '\n');
494 
495         if (AddrSpace == MMOAddrSpace) {
496           Success = true;
497           break;
498         }
499       }
500 
501       CurrentIdx = LastIdx;
502       if (!Success && handleReject() == RejectAndGiveUp)
503         return false;
504       break;
505     }
506     case GIM_CheckMemoryAlignment: {
507       int64_t InsnID = MatchTable[CurrentIdx++];
508       int64_t MMOIdx = MatchTable[CurrentIdx++];
509       unsigned MinAlign = MatchTable[CurrentIdx++];
510 
511       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
512 
513       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
514         if (handleReject() == RejectAndGiveUp)
515           return false;
516         break;
517       }
518 
519       MachineMemOperand *MMO =
520           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
521       DEBUG_WITH_TYPE(TgtExecutor::getName(),
522                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
523                              << "(MIs[" << InsnID << "]->memoperands() + "
524                              << MMOIdx << ")->getAlignment() >= " << MinAlign
525                              << ")\n");
526       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
527         return false;
528 
529       break;
530     }
531     case GIM_CheckMemorySizeEqualTo: {
532       int64_t InsnID = MatchTable[CurrentIdx++];
533       int64_t MMOIdx = MatchTable[CurrentIdx++];
534       uint64_t Size = MatchTable[CurrentIdx++];
535 
536       DEBUG_WITH_TYPE(TgtExecutor::getName(),
537                       dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
538                              << InsnID << "]->memoperands() + " << MMOIdx
539                              << ", Size=" << Size << ")\n");
540       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
541 
542       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
543         if (handleReject() == RejectAndGiveUp)
544           return false;
545         break;
546       }
547 
548       MachineMemOperand *MMO =
549           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
550 
551       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
552                                                      << " bytes vs " << Size
553                                                      << " bytes\n");
554       if (MMO->getSize() != Size)
555         if (handleReject() == RejectAndGiveUp)
556           return false;
557 
558       break;
559     }
560     case GIM_CheckMemorySizeEqualToLLT:
561     case GIM_CheckMemorySizeLessThanLLT:
562     case GIM_CheckMemorySizeGreaterThanLLT: {
563       int64_t InsnID = MatchTable[CurrentIdx++];
564       int64_t MMOIdx = MatchTable[CurrentIdx++];
565       int64_t OpIdx = MatchTable[CurrentIdx++];
566 
567       DEBUG_WITH_TYPE(
568           TgtExecutor::getName(),
569           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
570                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
571                      : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
572                          ? "GreaterThan"
573                          : "LessThan")
574                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
575                  << ", OpIdx=" << OpIdx << ")\n");
576       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577 
578       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579       if (!MO.isReg()) {
580         DEBUG_WITH_TYPE(TgtExecutor::getName(),
581                         dbgs() << CurrentIdx << ": Not a register\n");
582         if (handleReject() == RejectAndGiveUp)
583           return false;
584         break;
585       }
586 
587       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
588         if (handleReject() == RejectAndGiveUp)
589           return false;
590         break;
591       }
592 
593       MachineMemOperand *MMO =
594           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
595 
596       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
597       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
598           MMO->getSizeInBits() != Size) {
599         if (handleReject() == RejectAndGiveUp)
600           return false;
601       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
602                  MMO->getSizeInBits() >= Size) {
603         if (handleReject() == RejectAndGiveUp)
604           return false;
605       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
606                  MMO->getSizeInBits() <= Size)
607         if (handleReject() == RejectAndGiveUp)
608           return false;
609 
610       break;
611     }
612     case GIM_CheckType: {
613       int64_t InsnID = MatchTable[CurrentIdx++];
614       int64_t OpIdx = MatchTable[CurrentIdx++];
615       int64_t TypeID = MatchTable[CurrentIdx++];
616       DEBUG_WITH_TYPE(TgtExecutor::getName(),
617                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
618                              << "]->getOperand(" << OpIdx
619                              << "), TypeID=" << TypeID << ")\n");
620       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
622       if (!MO.isReg() ||
623           MRI.getType(MO.getReg()) != ExecInfo.TypeObjects[TypeID]) {
624         if (handleReject() == RejectAndGiveUp)
625           return false;
626       }
627       break;
628     }
629     case GIM_CheckPointerToAny: {
630       int64_t InsnID = MatchTable[CurrentIdx++];
631       int64_t OpIdx = MatchTable[CurrentIdx++];
632       uint64_t SizeInBits = MatchTable[CurrentIdx++];
633 
634       DEBUG_WITH_TYPE(TgtExecutor::getName(),
635                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
636                              << InsnID << "]->getOperand(" << OpIdx
637                              << "), SizeInBits=" << SizeInBits << ")\n");
638       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
639       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
640       const LLT Ty = MRI.getType(MO.getReg());
641 
642       // iPTR must be looked up in the target.
643       if (SizeInBits == 0) {
644         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
645         const unsigned AddrSpace = Ty.getAddressSpace();
646         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
647       }
648 
649       assert(SizeInBits != 0 && "Pointer size must be known");
650 
651       if (MO.isReg()) {
652         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
653           if (handleReject() == RejectAndGiveUp)
654             return false;
655       } else if (handleReject() == RejectAndGiveUp)
656         return false;
657 
658       break;
659     }
660     case GIM_RecordNamedOperand: {
661       int64_t InsnID = MatchTable[CurrentIdx++];
662       int64_t OpIdx = MatchTable[CurrentIdx++];
663       uint64_t StoreIdx = MatchTable[CurrentIdx++];
664 
665       DEBUG_WITH_TYPE(TgtExecutor::getName(),
666                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
667                              << InsnID << "]->getOperand(" << OpIdx
668                              << "), StoreIdx=" << StoreIdx << ")\n");
669       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
670       assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
671       State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
672       break;
673     }
674     case GIM_CheckRegBankForClass: {
675       int64_t InsnID = MatchTable[CurrentIdx++];
676       int64_t OpIdx = MatchTable[CurrentIdx++];
677       int64_t RCEnum = MatchTable[CurrentIdx++];
678       DEBUG_WITH_TYPE(TgtExecutor::getName(),
679                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
680                              << InsnID << "]->getOperand(" << OpIdx
681                              << "), RCEnum=" << RCEnum << ")\n");
682       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
683       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
684       if (!MO.isReg() ||
685           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
686                                       MRI.getType(MO.getReg())) !=
687               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
688         if (handleReject() == RejectAndGiveUp)
689           return false;
690       }
691       break;
692     }
693 
694     case GIM_CheckComplexPattern: {
695       int64_t InsnID = MatchTable[CurrentIdx++];
696       int64_t OpIdx = MatchTable[CurrentIdx++];
697       int64_t RendererID = MatchTable[CurrentIdx++];
698       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
699       DEBUG_WITH_TYPE(TgtExecutor::getName(),
700                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
701                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
702                              << "]->getOperand(" << OpIdx
703                              << "), ComplexPredicateID=" << ComplexPredicateID
704                              << ")\n");
705       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
706       // FIXME: Use std::invoke() when it's available.
707       ComplexRendererFns Renderer =
708           (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
709               State.MIs[InsnID]->getOperand(OpIdx));
710       if (Renderer)
711         State.Renderers[RendererID] = *Renderer;
712       else if (handleReject() == RejectAndGiveUp)
713         return false;
714       break;
715     }
716 
717     case GIM_CheckConstantInt: {
718       int64_t InsnID = MatchTable[CurrentIdx++];
719       int64_t OpIdx = MatchTable[CurrentIdx++];
720       int64_t Value = MatchTable[CurrentIdx++];
721       DEBUG_WITH_TYPE(TgtExecutor::getName(),
722                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
723                              << InsnID << "]->getOperand(" << OpIdx
724                              << "), Value=" << Value << ")\n");
725       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
726       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
727       if (MO.isReg()) {
728         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
729         LLT Ty = MRI.getType(MO.getReg());
730         Value = SignExtend64(Value, Ty.getSizeInBits());
731 
732         if (!isOperandImmEqual(MO, Value, MRI)) {
733           if (handleReject() == RejectAndGiveUp)
734             return false;
735         }
736       } else if (handleReject() == RejectAndGiveUp)
737         return false;
738 
739       break;
740     }
741 
742     case GIM_CheckLiteralInt: {
743       int64_t InsnID = MatchTable[CurrentIdx++];
744       int64_t OpIdx = MatchTable[CurrentIdx++];
745       int64_t Value = MatchTable[CurrentIdx++];
746       DEBUG_WITH_TYPE(TgtExecutor::getName(),
747                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
748                              << InsnID << "]->getOperand(" << OpIdx
749                              << "), Value=" << Value << ")\n");
750       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
751       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
752       if (MO.isImm() && MO.getImm() == Value)
753         break;
754 
755       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
756         break;
757 
758       if (handleReject() == RejectAndGiveUp)
759         return false;
760 
761       break;
762     }
763 
764     case GIM_CheckIntrinsicID: {
765       int64_t InsnID = MatchTable[CurrentIdx++];
766       int64_t OpIdx = MatchTable[CurrentIdx++];
767       int64_t Value = MatchTable[CurrentIdx++];
768       DEBUG_WITH_TYPE(TgtExecutor::getName(),
769                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
770                              << InsnID << "]->getOperand(" << OpIdx
771                              << "), Value=" << Value << ")\n");
772       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
773       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
774       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
775         if (handleReject() == RejectAndGiveUp)
776           return false;
777       break;
778     }
779     case GIM_CheckCmpPredicate: {
780       int64_t InsnID = MatchTable[CurrentIdx++];
781       int64_t OpIdx = MatchTable[CurrentIdx++];
782       int64_t Value = MatchTable[CurrentIdx++];
783       DEBUG_WITH_TYPE(TgtExecutor::getName(),
784                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
785                              << InsnID << "]->getOperand(" << OpIdx
786                              << "), Value=" << Value << ")\n");
787       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
788       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
789       if (!MO.isPredicate() || MO.getPredicate() != Value)
790         if (handleReject() == RejectAndGiveUp)
791           return false;
792       break;
793     }
794     case GIM_CheckIsMBB: {
795       int64_t InsnID = MatchTable[CurrentIdx++];
796       int64_t OpIdx = MatchTable[CurrentIdx++];
797       DEBUG_WITH_TYPE(TgtExecutor::getName(),
798                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
799                              << "]->getOperand(" << OpIdx << "))\n");
800       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
801       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
802         if (handleReject() == RejectAndGiveUp)
803           return false;
804       }
805       break;
806     }
807     case GIM_CheckIsImm: {
808       int64_t InsnID = MatchTable[CurrentIdx++];
809       int64_t OpIdx = MatchTable[CurrentIdx++];
810       DEBUG_WITH_TYPE(TgtExecutor::getName(),
811                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
812                              << "]->getOperand(" << OpIdx << "))\n");
813       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
814       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
815         if (handleReject() == RejectAndGiveUp)
816           return false;
817       }
818       break;
819     }
820     case GIM_CheckIsSafeToFold: {
821       int64_t InsnID = MatchTable[CurrentIdx++];
822       DEBUG_WITH_TYPE(TgtExecutor::getName(),
823                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
824                              << InsnID << "])\n");
825       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
826       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
827         if (handleReject() == RejectAndGiveUp)
828           return false;
829       }
830       break;
831     }
832     case GIM_CheckIsSameOperand:
833     case GIM_CheckIsSameOperandIgnoreCopies: {
834       int64_t InsnID = MatchTable[CurrentIdx++];
835       int64_t OpIdx = MatchTable[CurrentIdx++];
836       int64_t OtherInsnID = MatchTable[CurrentIdx++];
837       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
838       DEBUG_WITH_TYPE(TgtExecutor::getName(),
839                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
840                              << InsnID << "][" << OpIdx << "], MIs["
841                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
842       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
843       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
844 
845       MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
846       MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
847 
848       if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
849         if (Op.isReg() && OtherOp.isReg()) {
850           if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
851               getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
852             break;
853         }
854       }
855 
856       if (!Op.isIdenticalTo(OtherOp)) {
857         if (handleReject() == RejectAndGiveUp)
858           return false;
859       }
860       break;
861     }
862     case GIM_Reject:
863       DEBUG_WITH_TYPE(TgtExecutor::getName(),
864                       dbgs() << CurrentIdx << ": GIM_Reject\n");
865       if (handleReject() == RejectAndGiveUp)
866         return false;
867       break;
868     case GIR_MutateOpcode: {
869       int64_t OldInsnID = MatchTable[CurrentIdx++];
870       uint64_t NewInsnID = MatchTable[CurrentIdx++];
871       int64_t NewOpcode = MatchTable[CurrentIdx++];
872       if (NewInsnID >= OutMIs.size())
873         OutMIs.resize(NewInsnID + 1);
874 
875       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
876                                               State.MIs[OldInsnID]);
877       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
878       DEBUG_WITH_TYPE(TgtExecutor::getName(),
879                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
880                              << NewInsnID << "], MIs[" << OldInsnID << "], "
881                              << NewOpcode << ")\n");
882       break;
883     }
884 
885     case GIR_BuildMI: {
886       uint64_t NewInsnID = MatchTable[CurrentIdx++];
887       int64_t Opcode = MatchTable[CurrentIdx++];
888       if (NewInsnID >= OutMIs.size())
889         OutMIs.resize(NewInsnID + 1);
890 
891       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
892                                   MIMetadata(*State.MIs[0]), TII.get(Opcode));
893       DEBUG_WITH_TYPE(TgtExecutor::getName(),
894                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
895                              << NewInsnID << "], " << Opcode << ")\n");
896       break;
897     }
898 
899     case GIR_Copy: {
900       int64_t NewInsnID = MatchTable[CurrentIdx++];
901       int64_t OldInsnID = MatchTable[CurrentIdx++];
902       int64_t OpIdx = MatchTable[CurrentIdx++];
903       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
904       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
905       DEBUG_WITH_TYPE(TgtExecutor::getName(),
906                       dbgs()
907                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
908                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
909       break;
910     }
911 
912     case GIR_CopyOrAddZeroReg: {
913       int64_t NewInsnID = MatchTable[CurrentIdx++];
914       int64_t OldInsnID = MatchTable[CurrentIdx++];
915       int64_t OpIdx = MatchTable[CurrentIdx++];
916       int64_t ZeroReg = MatchTable[CurrentIdx++];
917       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
918       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
919       if (isOperandImmEqual(MO, 0, MRI))
920         OutMIs[NewInsnID].addReg(ZeroReg);
921       else
922         OutMIs[NewInsnID].add(MO);
923       DEBUG_WITH_TYPE(TgtExecutor::getName(),
924                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
925                              << NewInsnID << "], MIs[" << OldInsnID << "], "
926                              << OpIdx << ", " << ZeroReg << ")\n");
927       break;
928     }
929 
930     case GIR_CopySubReg: {
931       int64_t NewInsnID = MatchTable[CurrentIdx++];
932       int64_t OldInsnID = MatchTable[CurrentIdx++];
933       int64_t OpIdx = MatchTable[CurrentIdx++];
934       int64_t SubRegIdx = MatchTable[CurrentIdx++];
935       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
936       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
937                                0, SubRegIdx);
938       DEBUG_WITH_TYPE(TgtExecutor::getName(),
939                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
940                              << NewInsnID << "], MIs[" << OldInsnID << "], "
941                              << OpIdx << ", " << SubRegIdx << ")\n");
942       break;
943     }
944 
945     case GIR_AddImplicitDef: {
946       int64_t InsnID = MatchTable[CurrentIdx++];
947       int64_t RegNum = MatchTable[CurrentIdx++];
948       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
949       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
950       DEBUG_WITH_TYPE(TgtExecutor::getName(),
951                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
952                              << InsnID << "], " << RegNum << ")\n");
953       break;
954     }
955 
956     case GIR_AddImplicitUse: {
957       int64_t InsnID = MatchTable[CurrentIdx++];
958       int64_t RegNum = MatchTable[CurrentIdx++];
959       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
960       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
961       DEBUG_WITH_TYPE(TgtExecutor::getName(),
962                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
963                              << InsnID << "], " << RegNum << ")\n");
964       break;
965     }
966 
967     case GIR_AddRegister: {
968       int64_t InsnID = MatchTable[CurrentIdx++];
969       int64_t RegNum = MatchTable[CurrentIdx++];
970       uint64_t RegFlags = MatchTable[CurrentIdx++];
971       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
972       OutMIs[InsnID].addReg(RegNum, RegFlags);
973       DEBUG_WITH_TYPE(TgtExecutor::getName(),
974                       dbgs()
975                           << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
976                           << "], " << RegNum << ", " << RegFlags << ")\n");
977       break;
978     }
979 
980     case GIR_AddTempRegister:
981     case GIR_AddTempSubRegister: {
982       int64_t InsnID = MatchTable[CurrentIdx++];
983       int64_t TempRegID = MatchTable[CurrentIdx++];
984       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
985       unsigned SubReg = 0;
986       if (MatcherOpcode == GIR_AddTempSubRegister)
987         SubReg = MatchTable[CurrentIdx++];
988 
989       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
990 
991       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
992                             SubReg);
993       DEBUG_WITH_TYPE(
994           TgtExecutor::getName(),
995           dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
996                  << "], TempRegisters[" << TempRegID << "]";
997           if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
998           dbgs() << ", " << TempRegFlags << ")\n");
999       break;
1000     }
1001 
1002     case GIR_AddImm: {
1003       int64_t InsnID = MatchTable[CurrentIdx++];
1004       int64_t Imm = MatchTable[CurrentIdx++];
1005       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1006       OutMIs[InsnID].addImm(Imm);
1007       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1008                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1009                              << "], " << Imm << ")\n");
1010       break;
1011     }
1012 
1013     case GIR_ComplexRenderer: {
1014       int64_t InsnID = MatchTable[CurrentIdx++];
1015       int64_t RendererID = MatchTable[CurrentIdx++];
1016       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1017       for (const auto &RenderOpFn : State.Renderers[RendererID])
1018         RenderOpFn(OutMIs[InsnID]);
1019       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1020                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1021                              << InsnID << "], " << RendererID << ")\n");
1022       break;
1023     }
1024     case GIR_ComplexSubOperandRenderer: {
1025       int64_t InsnID = MatchTable[CurrentIdx++];
1026       int64_t RendererID = MatchTable[CurrentIdx++];
1027       int64_t RenderOpID = MatchTable[CurrentIdx++];
1028       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1029       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1030       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1031                       dbgs() << CurrentIdx
1032                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
1033                              << InsnID << "], " << RendererID << ", "
1034                              << RenderOpID << ")\n");
1035       break;
1036     }
1037     case GIR_ComplexSubOperandSubRegRenderer: {
1038       int64_t InsnID = MatchTable[CurrentIdx++];
1039       int64_t RendererID = MatchTable[CurrentIdx++];
1040       int64_t RenderOpID = MatchTable[CurrentIdx++];
1041       int64_t SubRegIdx = MatchTable[CurrentIdx++];
1042       MachineInstrBuilder &MI = OutMIs[InsnID];
1043       assert(MI && "Attempted to add to undefined instruction");
1044       State.Renderers[RendererID][RenderOpID](MI);
1045       MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1046       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1047                       dbgs() << CurrentIdx
1048                              << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1049                              << InsnID << "], " << RendererID << ", "
1050                              << RenderOpID << ", " << SubRegIdx << ")\n");
1051       break;
1052     }
1053 
1054     case GIR_CopyConstantAsSImm: {
1055       int64_t NewInsnID = MatchTable[CurrentIdx++];
1056       int64_t OldInsnID = MatchTable[CurrentIdx++];
1057       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1058       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1059              "Expected G_CONSTANT");
1060       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1061         OutMIs[NewInsnID].addImm(
1062             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1063       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1064         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1065       else
1066         llvm_unreachable("Expected Imm or CImm operand");
1067       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1068                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1069                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1070       break;
1071     }
1072 
1073     // TODO: Needs a test case once we have a pattern that uses this.
1074     case GIR_CopyFConstantAsFPImm: {
1075       int64_t NewInsnID = MatchTable[CurrentIdx++];
1076       int64_t OldInsnID = MatchTable[CurrentIdx++];
1077       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1078       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1079              "Expected G_FCONSTANT");
1080       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1081         OutMIs[NewInsnID].addFPImm(
1082             State.MIs[OldInsnID]->getOperand(1).getFPImm());
1083       else
1084         llvm_unreachable("Expected FPImm operand");
1085       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1086                       dbgs()
1087                           << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1088                           << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1089       break;
1090     }
1091 
1092     case GIR_CustomRenderer: {
1093       int64_t InsnID = MatchTable[CurrentIdx++];
1094       int64_t OldInsnID = MatchTable[CurrentIdx++];
1095       int64_t RendererFnID = MatchTable[CurrentIdx++];
1096       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1097       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1098                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1099                              << InsnID << "], MIs[" << OldInsnID << "], "
1100                              << RendererFnID << ")\n");
1101       (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1102           OutMIs[InsnID], *State.MIs[OldInsnID],
1103           -1); // Not a source operand of the old instruction.
1104       break;
1105     }
1106     case GIR_CustomAction: {
1107       int64_t FnID = MatchTable[CurrentIdx++];
1108       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1109                       dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
1110                              << ")\n");
1111       assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1112       runCustomAction(FnID, State);
1113       break;
1114     }
1115     case GIR_CustomOperandRenderer: {
1116       int64_t InsnID = MatchTable[CurrentIdx++];
1117       int64_t OldInsnID = MatchTable[CurrentIdx++];
1118       int64_t OpIdx = MatchTable[CurrentIdx++];
1119       int64_t RendererFnID = MatchTable[CurrentIdx++];
1120       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1121 
1122       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1123                       dbgs() << CurrentIdx
1124                              << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1125                              << "], MIs[" << OldInsnID << "]->getOperand("
1126                              << OpIdx << "), " << RendererFnID << ")\n");
1127       (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1128           OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1129       break;
1130     }
1131     case GIR_ConstrainOperandRC: {
1132       int64_t InsnID = MatchTable[CurrentIdx++];
1133       int64_t OpIdx = MatchTable[CurrentIdx++];
1134       int64_t RCEnum = MatchTable[CurrentIdx++];
1135       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1136       MachineInstr &I = *OutMIs[InsnID].getInstr();
1137       MachineFunction &MF = *I.getParent()->getParent();
1138       MachineRegisterInfo &MRI = MF.getRegInfo();
1139       const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1140       MachineOperand &MO = I.getOperand(OpIdx);
1141       constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1142       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1143                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1144                              << InsnID << "], " << OpIdx << ", " << RCEnum
1145                              << ")\n");
1146       break;
1147     }
1148 
1149     case GIR_ConstrainSelectedInstOperands: {
1150       int64_t InsnID = MatchTable[CurrentIdx++];
1151       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1152       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1153                                        RBI);
1154       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1155                       dbgs() << CurrentIdx
1156                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1157                              << InsnID << "])\n");
1158       break;
1159     }
1160 
1161     case GIR_MergeMemOperands: {
1162       int64_t InsnID = MatchTable[CurrentIdx++];
1163       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1164 
1165       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1166                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1167                              << InsnID << "]");
1168       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1169       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1170              GIU_MergeMemOperands_EndOfList) {
1171         DEBUG_WITH_TYPE(TgtExecutor::getName(),
1172                         dbgs() << ", MIs[" << MergeInsnID << "]");
1173         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1174           OutMIs[InsnID].addMemOperand(MMO);
1175       }
1176       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1177       break;
1178     }
1179 
1180     case GIR_EraseFromParent: {
1181       int64_t InsnID = MatchTable[CurrentIdx++];
1182       assert(State.MIs[InsnID] &&
1183              "Attempted to erase an undefined instruction");
1184       State.MIs[InsnID]->eraseFromParent();
1185       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1186                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1187                              << InsnID << "])\n");
1188       break;
1189     }
1190 
1191     case GIR_MakeTempReg: {
1192       int64_t TempRegID = MatchTable[CurrentIdx++];
1193       int64_t TypeID = MatchTable[CurrentIdx++];
1194 
1195       State.TempRegisters[TempRegID] =
1196           MRI.createGenericVirtualRegister(ExecInfo.TypeObjects[TypeID]);
1197       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1198                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1199                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1200       break;
1201     }
1202 
1203     case GIR_Coverage: {
1204       int64_t RuleID = MatchTable[CurrentIdx++];
1205       assert(CoverageInfo);
1206       CoverageInfo->setCovered(RuleID);
1207 
1208       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1209                                                      << ": GIR_Coverage("
1210                                                      << RuleID << ")");
1211       break;
1212     }
1213 
1214     case GIR_Done:
1215       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1216                       dbgs() << CurrentIdx << ": GIR_Done\n");
1217       propagateFlags(OutMIs);
1218       return true;
1219     default:
1220       llvm_unreachable("Unexpected command");
1221     }
1222   }
1223 }
1224 
1225 } // end namespace llvm
1226 
1227 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1228