1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file declares the API for the instruction selector.
10 /// This class is responsible for selecting machine instructions.
11 /// It's implemented by the target. It's used by the InstructionSelect pass.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21 #include "llvm/CodeGen/GlobalISel/Utils.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <cstddef>
35 #include <cstdint>
36 
37 namespace llvm {
38 
39 /// GlobalISel PatFrag Predicates
40 enum {
41   GIPFP_I64_Invalid = 0,
42   GIPFP_APInt_Invalid = 0,
43   GIPFP_APFloat_Invalid = 0,
44   GIPFP_MI_Invalid = 0,
45 };
46 
47 template <class TgtInstructionSelector, class PredicateBitset,
48           class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)49 bool InstructionSelector::executeMatchTable(
50     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52         &ISelInfo,
53     const int64_t *MatchTable, const TargetInstrInfo &TII,
54     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56     CodeGenCoverage &CoverageInfo) const {
57 
58   uint64_t CurrentIdx = 0;
59   SmallVector<uint64_t, 4> OnFailResumeAt;
60 
61   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63 
64   const uint16_t Flags = State.MIs[0]->getFlags();
65 
66   enum RejectAction { RejectAndGiveUp, RejectAndResume };
67   auto handleReject = [&]() -> RejectAction {
68     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69                     dbgs() << CurrentIdx << ": Rejected\n");
70     if (OnFailResumeAt.empty())
71       return RejectAndGiveUp;
72     CurrentIdx = OnFailResumeAt.pop_back_val();
73     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
74                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75                            << OnFailResumeAt.size() << " try-blocks remain)\n");
76     return RejectAndResume;
77   };
78 
79   auto propagateFlags = [=](NewMIVector &OutMIs) {
80     for (auto MIB : OutMIs) {
81       // Set the NoFPExcept flag when no original matched instruction could
82       // raise an FP exception, but the new instruction potentially might.
83       uint16_t MIBFlags = Flags;
84       if (NoFPException && MIB->mayRaiseFPException())
85         MIBFlags |= MachineInstr::NoFPExcept;
86       MIB.setMIFlags(MIBFlags);
87     }
88 
89     return true;
90   };
91 
92   while (true) {
93     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
94     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
95     switch (MatcherOpcode) {
96     case GIM_Try: {
97       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
98                       dbgs() << CurrentIdx << ": Begin try-block\n");
99       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
100       break;
101     }
102 
103     case GIM_RecordInsn: {
104       int64_t NewInsnID = MatchTable[CurrentIdx++];
105       int64_t InsnID = MatchTable[CurrentIdx++];
106       int64_t OpIdx = MatchTable[CurrentIdx++];
107 
108       // As an optimisation we require that MIs[0] is always the root. Refuse
109       // any attempt to modify it.
110       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
111 
112       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
113       if (!MO.isReg()) {
114         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
115                         dbgs() << CurrentIdx << ": Not a register\n");
116         if (handleReject() == RejectAndGiveUp)
117           return false;
118         break;
119       }
120       if (Register::isPhysicalRegister(MO.getReg())) {
121         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
122                         dbgs() << CurrentIdx << ": Is a physical register\n");
123         if (handleReject() == RejectAndGiveUp)
124           return false;
125         break;
126       }
127 
128       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
129       if ((size_t)NewInsnID < State.MIs.size())
130         State.MIs[NewInsnID] = NewMI;
131       else {
132         assert((size_t)NewInsnID == State.MIs.size() &&
133                "Expected to store MIs in order");
134         State.MIs.push_back(NewMI);
135       }
136       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
137                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
138                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
139                              << ")\n");
140       break;
141     }
142 
143     case GIM_CheckFeatures: {
144       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146                       dbgs() << CurrentIdx
147                              << ": GIM_CheckFeatures(ExpectedBitsetID="
148                              << ExpectedBitsetID << ")\n");
149       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
150           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
151         if (handleReject() == RejectAndGiveUp)
152           return false;
153       }
154       break;
155     }
156 
157     case GIM_CheckOpcode:
158     case GIM_CheckOpcodeIsEither: {
159       int64_t InsnID = MatchTable[CurrentIdx++];
160       int64_t Expected0 = MatchTable[CurrentIdx++];
161       int64_t Expected1 = -1;
162       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
163         Expected1 = MatchTable[CurrentIdx++];
164 
165       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
166       unsigned Opcode = State.MIs[InsnID]->getOpcode();
167 
168       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
169         dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
170         << "], ExpectedOpcode=" << Expected0;
171         if (MatcherOpcode == GIM_CheckOpcodeIsEither)
172           dbgs() << " || " << Expected1;
173         dbgs() << ") // Got=" << Opcode << "\n";
174       );
175 
176       if (Opcode != Expected0 && Opcode != Expected1) {
177         if (handleReject() == RejectAndGiveUp)
178           return false;
179       }
180       break;
181     }
182     case GIM_SwitchOpcode: {
183       int64_t InsnID = MatchTable[CurrentIdx++];
184       int64_t LowerBound = MatchTable[CurrentIdx++];
185       int64_t UpperBound = MatchTable[CurrentIdx++];
186       int64_t Default = MatchTable[CurrentIdx++];
187 
188       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
189       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
190 
191       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
192         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
193                << LowerBound << ", " << UpperBound << "), Default=" << Default
194                << ", JumpTable...) // Got=" << Opcode << "\n";
195       });
196       if (Opcode < LowerBound || UpperBound <= Opcode) {
197         CurrentIdx = Default;
198         break;
199       }
200       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
201       if (!CurrentIdx) {
202         CurrentIdx = Default;
203         break;
204       }
205       OnFailResumeAt.push_back(Default);
206       break;
207     }
208 
209     case GIM_SwitchType: {
210       int64_t InsnID = MatchTable[CurrentIdx++];
211       int64_t OpIdx = MatchTable[CurrentIdx++];
212       int64_t LowerBound = MatchTable[CurrentIdx++];
213       int64_t UpperBound = MatchTable[CurrentIdx++];
214       int64_t Default = MatchTable[CurrentIdx++];
215 
216       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
217       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
218 
219       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
220         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
221                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
222                << UpperBound << "), Default=" << Default
223                << ", JumpTable...) // Got=";
224         if (!MO.isReg())
225           dbgs() << "Not a VReg\n";
226         else
227           dbgs() << MRI.getType(MO.getReg()) << "\n";
228       });
229       if (!MO.isReg()) {
230         CurrentIdx = Default;
231         break;
232       }
233       const LLT Ty = MRI.getType(MO.getReg());
234       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
235       if (TyI == ISelInfo.TypeIDMap.end()) {
236         CurrentIdx = Default;
237         break;
238       }
239       const int64_t TypeID = TyI->second;
240       if (TypeID < LowerBound || UpperBound <= TypeID) {
241         CurrentIdx = Default;
242         break;
243       }
244       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
245       if (!CurrentIdx) {
246         CurrentIdx = Default;
247         break;
248       }
249       OnFailResumeAt.push_back(Default);
250       break;
251     }
252 
253     case GIM_CheckNumOperands: {
254       int64_t InsnID = MatchTable[CurrentIdx++];
255       int64_t Expected = MatchTable[CurrentIdx++];
256       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
257                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
258                              << InsnID << "], Expected=" << Expected << ")\n");
259       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
260       if (State.MIs[InsnID]->getNumOperands() != Expected) {
261         if (handleReject() == RejectAndGiveUp)
262           return false;
263       }
264       break;
265     }
266     case GIM_CheckI64ImmPredicate: {
267       int64_t InsnID = MatchTable[CurrentIdx++];
268       int64_t Predicate = MatchTable[CurrentIdx++];
269       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
270                       dbgs()
271                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
272                           << InsnID << "], Predicate=" << Predicate << ")\n");
273       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
274       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
275              "Expected G_CONSTANT");
276       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
277       int64_t Value = 0;
278       if (State.MIs[InsnID]->getOperand(1).isCImm())
279         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
280       else if (State.MIs[InsnID]->getOperand(1).isImm())
281         Value = State.MIs[InsnID]->getOperand(1).getImm();
282       else
283         llvm_unreachable("Expected Imm or CImm operand");
284 
285       if (!testImmPredicate_I64(Predicate, Value))
286         if (handleReject() == RejectAndGiveUp)
287           return false;
288       break;
289     }
290     case GIM_CheckAPIntImmPredicate: {
291       int64_t InsnID = MatchTable[CurrentIdx++];
292       int64_t Predicate = MatchTable[CurrentIdx++];
293       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
294                       dbgs()
295                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
296                           << InsnID << "], Predicate=" << Predicate << ")\n");
297       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
298       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
299              "Expected G_CONSTANT");
300       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
301       APInt Value;
302       if (State.MIs[InsnID]->getOperand(1).isCImm())
303         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
304       else
305         llvm_unreachable("Expected Imm or CImm operand");
306 
307       if (!testImmPredicate_APInt(Predicate, Value))
308         if (handleReject() == RejectAndGiveUp)
309           return false;
310       break;
311     }
312     case GIM_CheckAPFloatImmPredicate: {
313       int64_t InsnID = MatchTable[CurrentIdx++];
314       int64_t Predicate = MatchTable[CurrentIdx++];
315       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
316                       dbgs()
317                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
318                           << InsnID << "], Predicate=" << Predicate << ")\n");
319       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
320       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
321              "Expected G_FCONSTANT");
322       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
323       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
324       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
325 
326       if (!testImmPredicate_APFloat(Predicate, Value))
327         if (handleReject() == RejectAndGiveUp)
328           return false;
329       break;
330     }
331     case GIM_CheckIsBuildVectorAllOnes:
332     case GIM_CheckIsBuildVectorAllZeros: {
333       int64_t InsnID = MatchTable[CurrentIdx++];
334 
335       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
336                       dbgs() << CurrentIdx
337                              << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
338                              << InsnID << "])\n");
339       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
340 
341       const MachineInstr *MI = State.MIs[InsnID];
342       assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
343               MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
344              "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
345 
346       if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
347         if (!isBuildVectorAllOnes(*MI, MRI)) {
348           if (handleReject() == RejectAndGiveUp)
349             return false;
350         }
351       } else {
352         if (!isBuildVectorAllZeros(*MI, MRI)) {
353           if (handleReject() == RejectAndGiveUp)
354             return false;
355         }
356       }
357 
358       break;
359     }
360     case GIM_CheckCxxInsnPredicate: {
361       int64_t InsnID = MatchTable[CurrentIdx++];
362       int64_t Predicate = MatchTable[CurrentIdx++];
363       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
364                       dbgs()
365                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
366                           << InsnID << "], Predicate=" << Predicate << ")\n");
367       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
368       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
369 
370       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
371                               State.RecordedOperands))
372         if (handleReject() == RejectAndGiveUp)
373           return false;
374       break;
375     }
376     case GIM_CheckAtomicOrdering: {
377       int64_t InsnID = MatchTable[CurrentIdx++];
378       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
379       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
381                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
382       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
383       if (!State.MIs[InsnID]->hasOneMemOperand())
384         if (handleReject() == RejectAndGiveUp)
385           return false;
386 
387       for (const auto &MMO : State.MIs[InsnID]->memoperands())
388         if (MMO->getOrdering() != Ordering)
389           if (handleReject() == RejectAndGiveUp)
390             return false;
391       break;
392     }
393     case GIM_CheckAtomicOrderingOrStrongerThan: {
394       int64_t InsnID = MatchTable[CurrentIdx++];
395       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
396       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
397                       dbgs() << CurrentIdx
398                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
399                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
400       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
401       if (!State.MIs[InsnID]->hasOneMemOperand())
402         if (handleReject() == RejectAndGiveUp)
403           return false;
404 
405       for (const auto &MMO : State.MIs[InsnID]->memoperands())
406         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
407           if (handleReject() == RejectAndGiveUp)
408             return false;
409       break;
410     }
411     case GIM_CheckAtomicOrderingWeakerThan: {
412       int64_t InsnID = MatchTable[CurrentIdx++];
413       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
414       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
415                       dbgs() << CurrentIdx
416                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
417                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
418       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
419       if (!State.MIs[InsnID]->hasOneMemOperand())
420         if (handleReject() == RejectAndGiveUp)
421           return false;
422 
423       for (const auto &MMO : State.MIs[InsnID]->memoperands())
424         if (!isStrongerThan(Ordering, MMO->getOrdering()))
425           if (handleReject() == RejectAndGiveUp)
426             return false;
427       break;
428     }
429     case GIM_CheckMemoryAddressSpace: {
430       int64_t InsnID = MatchTable[CurrentIdx++];
431       int64_t MMOIdx = MatchTable[CurrentIdx++];
432       // This accepts a list of possible address spaces.
433       const int NumAddrSpace = MatchTable[CurrentIdx++];
434 
435       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
436         if (handleReject() == RejectAndGiveUp)
437           return false;
438         break;
439       }
440 
441       // Need to still jump to the end of the list of address spaces if we find
442       // a match earlier.
443       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
444 
445       const MachineMemOperand *MMO
446         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
447       const unsigned MMOAddrSpace = MMO->getAddrSpace();
448 
449       bool Success = false;
450       for (int I = 0; I != NumAddrSpace; ++I) {
451         unsigned AddrSpace = MatchTable[CurrentIdx++];
452         DEBUG_WITH_TYPE(
453           TgtInstructionSelector::getName(),
454           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
455                  << AddrSpace << '\n');
456 
457         if (AddrSpace == MMOAddrSpace) {
458           Success = true;
459           break;
460         }
461       }
462 
463       CurrentIdx = LastIdx;
464       if (!Success && handleReject() == RejectAndGiveUp)
465         return false;
466       break;
467     }
468     case GIM_CheckMemoryAlignment: {
469       int64_t InsnID = MatchTable[CurrentIdx++];
470       int64_t MMOIdx = MatchTable[CurrentIdx++];
471       unsigned MinAlign = MatchTable[CurrentIdx++];
472 
473       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
474 
475       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
476         if (handleReject() == RejectAndGiveUp)
477           return false;
478         break;
479       }
480 
481       MachineMemOperand *MMO
482         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
483       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
484                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
485                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
486                       << ")->getAlignment() >= " << MinAlign << ")\n");
487       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
488         return false;
489 
490       break;
491     }
492     case GIM_CheckMemorySizeEqualTo: {
493       int64_t InsnID = MatchTable[CurrentIdx++];
494       int64_t MMOIdx = MatchTable[CurrentIdx++];
495       uint64_t Size = MatchTable[CurrentIdx++];
496 
497       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498                       dbgs() << CurrentIdx
499                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
500                              << "]->memoperands() + " << MMOIdx
501                              << ", Size=" << Size << ")\n");
502       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
503 
504       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
505         if (handleReject() == RejectAndGiveUp)
506           return false;
507         break;
508       }
509 
510       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
511 
512       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
513                       dbgs() << MMO->getSize() << " bytes vs " << Size
514                              << " bytes\n");
515       if (MMO->getSize() != Size)
516         if (handleReject() == RejectAndGiveUp)
517           return false;
518 
519       break;
520     }
521     case GIM_CheckMemorySizeEqualToLLT:
522     case GIM_CheckMemorySizeLessThanLLT:
523     case GIM_CheckMemorySizeGreaterThanLLT: {
524       int64_t InsnID = MatchTable[CurrentIdx++];
525       int64_t MMOIdx = MatchTable[CurrentIdx++];
526       int64_t OpIdx = MatchTable[CurrentIdx++];
527 
528       DEBUG_WITH_TYPE(
529           TgtInstructionSelector::getName(),
530           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
531                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
532                          ? "EqualTo"
533                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
534                                ? "GreaterThan"
535                                : "LessThan")
536                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
537                  << ", OpIdx=" << OpIdx << ")\n");
538       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
539 
540       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
541       if (!MO.isReg()) {
542         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
543                         dbgs() << CurrentIdx << ": Not a register\n");
544         if (handleReject() == RejectAndGiveUp)
545           return false;
546         break;
547       }
548 
549       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
550         if (handleReject() == RejectAndGiveUp)
551           return false;
552         break;
553       }
554 
555       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
556 
557       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
558       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
559           MMO->getSizeInBits() != Size) {
560         if (handleReject() == RejectAndGiveUp)
561           return false;
562       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
563                  MMO->getSizeInBits() >= Size) {
564         if (handleReject() == RejectAndGiveUp)
565           return false;
566       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
567                  MMO->getSizeInBits() <= Size)
568         if (handleReject() == RejectAndGiveUp)
569           return false;
570 
571       break;
572     }
573     case GIM_CheckType: {
574       int64_t InsnID = MatchTable[CurrentIdx++];
575       int64_t OpIdx = MatchTable[CurrentIdx++];
576       int64_t TypeID = MatchTable[CurrentIdx++];
577       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
578                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
579                              << "]->getOperand(" << OpIdx
580                              << "), TypeID=" << TypeID << ")\n");
581       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
582       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
583       if (!MO.isReg() ||
584           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
585         if (handleReject() == RejectAndGiveUp)
586           return false;
587       }
588       break;
589     }
590     case GIM_CheckPointerToAny: {
591       int64_t InsnID = MatchTable[CurrentIdx++];
592       int64_t OpIdx = MatchTable[CurrentIdx++];
593       int64_t SizeInBits = MatchTable[CurrentIdx++];
594 
595       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
596                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
597                              << InsnID << "]->getOperand(" << OpIdx
598                              << "), SizeInBits=" << SizeInBits << ")\n");
599       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
600       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
601       const LLT Ty = MRI.getType(MO.getReg());
602 
603       // iPTR must be looked up in the target.
604       if (SizeInBits == 0) {
605         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
606         const unsigned AddrSpace = Ty.getAddressSpace();
607         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
608       }
609 
610       assert(SizeInBits != 0 && "Pointer size must be known");
611 
612       if (MO.isReg()) {
613         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
614           if (handleReject() == RejectAndGiveUp)
615             return false;
616       } else if (handleReject() == RejectAndGiveUp)
617         return false;
618 
619       break;
620     }
621     case GIM_RecordNamedOperand: {
622       int64_t InsnID = MatchTable[CurrentIdx++];
623       int64_t OpIdx = MatchTable[CurrentIdx++];
624       uint64_t StoreIdx = MatchTable[CurrentIdx++];
625 
626       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
627                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
628                              << InsnID << "]->getOperand(" << OpIdx
629                              << "), StoreIdx=" << StoreIdx << ")\n");
630       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
631       assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
632       State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
633       break;
634     }
635     case GIM_CheckRegBankForClass: {
636       int64_t InsnID = MatchTable[CurrentIdx++];
637       int64_t OpIdx = MatchTable[CurrentIdx++];
638       int64_t RCEnum = MatchTable[CurrentIdx++];
639       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
640                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
641                              << InsnID << "]->getOperand(" << OpIdx
642                              << "), RCEnum=" << RCEnum << ")\n");
643       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
644       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
645       if (!MO.isReg() ||
646           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
647                                       MRI.getType(MO.getReg())) !=
648               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
649         if (handleReject() == RejectAndGiveUp)
650           return false;
651       }
652       break;
653     }
654 
655     case GIM_CheckComplexPattern: {
656       int64_t InsnID = MatchTable[CurrentIdx++];
657       int64_t OpIdx = MatchTable[CurrentIdx++];
658       int64_t RendererID = MatchTable[CurrentIdx++];
659       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
660       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
661                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
662                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
663                              << "]->getOperand(" << OpIdx
664                              << "), ComplexPredicateID=" << ComplexPredicateID
665                              << ")\n");
666       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
667       // FIXME: Use std::invoke() when it's available.
668       ComplexRendererFns Renderer =
669           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
670               State.MIs[InsnID]->getOperand(OpIdx));
671       if (Renderer.hasValue())
672         State.Renderers[RendererID] = Renderer.getValue();
673       else
674         if (handleReject() == RejectAndGiveUp)
675           return false;
676       break;
677     }
678 
679     case GIM_CheckConstantInt: {
680       int64_t InsnID = MatchTable[CurrentIdx++];
681       int64_t OpIdx = MatchTable[CurrentIdx++];
682       int64_t Value = MatchTable[CurrentIdx++];
683       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
684                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
685                              << InsnID << "]->getOperand(" << OpIdx
686                              << "), Value=" << Value << ")\n");
687       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
688       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
689       if (MO.isReg()) {
690         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
691         LLT Ty = MRI.getType(MO.getReg());
692         Value = SignExtend64(Value, Ty.getSizeInBits());
693 
694         if (!isOperandImmEqual(MO, Value, MRI)) {
695           if (handleReject() == RejectAndGiveUp)
696             return false;
697         }
698       } else if (handleReject() == RejectAndGiveUp)
699         return false;
700 
701       break;
702     }
703 
704     case GIM_CheckLiteralInt: {
705       int64_t InsnID = MatchTable[CurrentIdx++];
706       int64_t OpIdx = MatchTable[CurrentIdx++];
707       int64_t Value = MatchTable[CurrentIdx++];
708       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
710                              << InsnID << "]->getOperand(" << OpIdx
711                              << "), Value=" << Value << ")\n");
712       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
713       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
714       if (MO.isImm() && MO.getImm() == Value)
715         break;
716 
717       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
718         break;
719 
720       if (handleReject() == RejectAndGiveUp)
721         return false;
722 
723       break;
724     }
725 
726     case GIM_CheckIntrinsicID: {
727       int64_t InsnID = MatchTable[CurrentIdx++];
728       int64_t OpIdx = MatchTable[CurrentIdx++];
729       int64_t Value = MatchTable[CurrentIdx++];
730       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
732                              << InsnID << "]->getOperand(" << OpIdx
733                              << "), Value=" << Value << ")\n");
734       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
735       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
736       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
737         if (handleReject() == RejectAndGiveUp)
738           return false;
739       break;
740     }
741     case GIM_CheckCmpPredicate: {
742       int64_t InsnID = MatchTable[CurrentIdx++];
743       int64_t OpIdx = MatchTable[CurrentIdx++];
744       int64_t Value = MatchTable[CurrentIdx++];
745       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
747                              << InsnID << "]->getOperand(" << OpIdx
748                              << "), Value=" << Value << ")\n");
749       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
750       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
751       if (!MO.isPredicate() || MO.getPredicate() != Value)
752         if (handleReject() == RejectAndGiveUp)
753           return false;
754       break;
755     }
756     case GIM_CheckIsMBB: {
757       int64_t InsnID = MatchTable[CurrentIdx++];
758       int64_t OpIdx = MatchTable[CurrentIdx++];
759       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
760                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
761                              << "]->getOperand(" << OpIdx << "))\n");
762       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
763       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
764         if (handleReject() == RejectAndGiveUp)
765           return false;
766       }
767       break;
768     }
769     case GIM_CheckIsImm: {
770       int64_t InsnID = MatchTable[CurrentIdx++];
771       int64_t OpIdx = MatchTable[CurrentIdx++];
772       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
773                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
774                              << "]->getOperand(" << OpIdx << "))\n");
775       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
776       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
777         if (handleReject() == RejectAndGiveUp)
778           return false;
779       }
780       break;
781     }
782     case GIM_CheckIsSafeToFold: {
783       int64_t InsnID = MatchTable[CurrentIdx++];
784       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
785                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
786                              << InsnID << "])\n");
787       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
788       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
789         if (handleReject() == RejectAndGiveUp)
790           return false;
791       }
792       break;
793     }
794     case GIM_CheckIsSameOperand: {
795       int64_t InsnID = MatchTable[CurrentIdx++];
796       int64_t OpIdx = MatchTable[CurrentIdx++];
797       int64_t OtherInsnID = MatchTable[CurrentIdx++];
798       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
799       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
800                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
801                              << InsnID << "][" << OpIdx << "], MIs["
802                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
803       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
804       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
805       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
806               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
807         if (handleReject() == RejectAndGiveUp)
808           return false;
809       }
810       break;
811     }
812     case GIM_Reject:
813       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
814                       dbgs() << CurrentIdx << ": GIM_Reject\n");
815       if (handleReject() == RejectAndGiveUp)
816         return false;
817       break;
818 
819     case GIR_MutateOpcode: {
820       int64_t OldInsnID = MatchTable[CurrentIdx++];
821       uint64_t NewInsnID = MatchTable[CurrentIdx++];
822       int64_t NewOpcode = MatchTable[CurrentIdx++];
823       if (NewInsnID >= OutMIs.size())
824         OutMIs.resize(NewInsnID + 1);
825 
826       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
827                                               State.MIs[OldInsnID]);
828       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
829       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
830                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
831                              << NewInsnID << "], MIs[" << OldInsnID << "], "
832                              << NewOpcode << ")\n");
833       break;
834     }
835 
836     case GIR_BuildMI: {
837       uint64_t NewInsnID = MatchTable[CurrentIdx++];
838       int64_t Opcode = MatchTable[CurrentIdx++];
839       if (NewInsnID >= OutMIs.size())
840         OutMIs.resize(NewInsnID + 1);
841 
842       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
843                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
844       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
846                              << NewInsnID << "], " << Opcode << ")\n");
847       break;
848     }
849 
850     case GIR_Copy: {
851       int64_t NewInsnID = MatchTable[CurrentIdx++];
852       int64_t OldInsnID = MatchTable[CurrentIdx++];
853       int64_t OpIdx = MatchTable[CurrentIdx++];
854       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
855       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
856       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
857                       dbgs()
858                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
859                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
860       break;
861     }
862 
863     case GIR_CopyOrAddZeroReg: {
864       int64_t NewInsnID = MatchTable[CurrentIdx++];
865       int64_t OldInsnID = MatchTable[CurrentIdx++];
866       int64_t OpIdx = MatchTable[CurrentIdx++];
867       int64_t ZeroReg = MatchTable[CurrentIdx++];
868       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
869       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
870       if (isOperandImmEqual(MO, 0, MRI))
871         OutMIs[NewInsnID].addReg(ZeroReg);
872       else
873         OutMIs[NewInsnID].add(MO);
874       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
875                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
876                              << NewInsnID << "], MIs[" << OldInsnID << "], "
877                              << OpIdx << ", " << ZeroReg << ")\n");
878       break;
879     }
880 
881     case GIR_CopySubReg: {
882       int64_t NewInsnID = MatchTable[CurrentIdx++];
883       int64_t OldInsnID = MatchTable[CurrentIdx++];
884       int64_t OpIdx = MatchTable[CurrentIdx++];
885       int64_t SubRegIdx = MatchTable[CurrentIdx++];
886       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
887       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
888                                0, SubRegIdx);
889       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
891                              << NewInsnID << "], MIs[" << OldInsnID << "], "
892                              << OpIdx << ", " << SubRegIdx << ")\n");
893       break;
894     }
895 
896     case GIR_AddImplicitDef: {
897       int64_t InsnID = MatchTable[CurrentIdx++];
898       int64_t RegNum = MatchTable[CurrentIdx++];
899       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
900       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
901       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
902                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
903                              << InsnID << "], " << RegNum << ")\n");
904       break;
905     }
906 
907     case GIR_AddImplicitUse: {
908       int64_t InsnID = MatchTable[CurrentIdx++];
909       int64_t RegNum = MatchTable[CurrentIdx++];
910       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
911       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
912       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
913                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
914                              << InsnID << "], " << RegNum << ")\n");
915       break;
916     }
917 
918     case GIR_AddRegister: {
919       int64_t InsnID = MatchTable[CurrentIdx++];
920       int64_t RegNum = MatchTable[CurrentIdx++];
921       uint64_t RegFlags = MatchTable[CurrentIdx++];
922       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
923       OutMIs[InsnID].addReg(RegNum, RegFlags);
924       DEBUG_WITH_TYPE(
925         TgtInstructionSelector::getName(),
926         dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
927         << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
928       break;
929     }
930 
931     case GIR_AddTempRegister:
932     case GIR_AddTempSubRegister: {
933       int64_t InsnID = MatchTable[CurrentIdx++];
934       int64_t TempRegID = MatchTable[CurrentIdx++];
935       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
936       unsigned SubReg = 0;
937       if (MatcherOpcode == GIR_AddTempSubRegister)
938         SubReg = MatchTable[CurrentIdx++];
939 
940       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
941 
942       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
943       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
944                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
945                              << InsnID << "], TempRegisters[" << TempRegID
946                              << "]";
947                       if (SubReg)
948                         dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
949                       dbgs() << ", " << TempRegFlags << ")\n");
950       break;
951     }
952 
953     case GIR_AddImm: {
954       int64_t InsnID = MatchTable[CurrentIdx++];
955       int64_t Imm = MatchTable[CurrentIdx++];
956       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
957       OutMIs[InsnID].addImm(Imm);
958       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
959                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
960                              << "], " << Imm << ")\n");
961       break;
962     }
963 
964     case GIR_ComplexRenderer: {
965       int64_t InsnID = MatchTable[CurrentIdx++];
966       int64_t RendererID = MatchTable[CurrentIdx++];
967       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
968       for (const auto &RenderOpFn : State.Renderers[RendererID])
969         RenderOpFn(OutMIs[InsnID]);
970       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
971                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
972                              << InsnID << "], " << RendererID << ")\n");
973       break;
974     }
975     case GIR_ComplexSubOperandRenderer: {
976       int64_t InsnID = MatchTable[CurrentIdx++];
977       int64_t RendererID = MatchTable[CurrentIdx++];
978       int64_t RenderOpID = MatchTable[CurrentIdx++];
979       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
980       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
981       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
982                       dbgs() << CurrentIdx
983                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
984                              << InsnID << "], " << RendererID << ", "
985                              << RenderOpID << ")\n");
986       break;
987     }
988 
989     case GIR_CopyConstantAsSImm: {
990       int64_t NewInsnID = MatchTable[CurrentIdx++];
991       int64_t OldInsnID = MatchTable[CurrentIdx++];
992       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
993       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
994       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
995         OutMIs[NewInsnID].addImm(
996             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
997       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
998         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
999       else
1000         llvm_unreachable("Expected Imm or CImm operand");
1001       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1002                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1003                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1004       break;
1005     }
1006 
1007     // TODO: Needs a test case once we have a pattern that uses this.
1008     case GIR_CopyFConstantAsFPImm: {
1009       int64_t NewInsnID = MatchTable[CurrentIdx++];
1010       int64_t OldInsnID = MatchTable[CurrentIdx++];
1011       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1012       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
1013       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1014         OutMIs[NewInsnID].addFPImm(
1015             State.MIs[OldInsnID]->getOperand(1).getFPImm());
1016       else
1017         llvm_unreachable("Expected FPImm operand");
1018       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1019                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1020                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1021       break;
1022     }
1023 
1024     case GIR_CustomRenderer: {
1025       int64_t InsnID = MatchTable[CurrentIdx++];
1026       int64_t OldInsnID = MatchTable[CurrentIdx++];
1027       int64_t RendererFnID = MatchTable[CurrentIdx++];
1028       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1029       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1030                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1031                              << InsnID << "], MIs[" << OldInsnID << "], "
1032                              << RendererFnID << ")\n");
1033       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
1034         OutMIs[InsnID], *State.MIs[OldInsnID],
1035         -1); // Not a source operand of the old instruction.
1036       break;
1037     }
1038     case GIR_CustomOperandRenderer: {
1039       int64_t InsnID = MatchTable[CurrentIdx++];
1040       int64_t OldInsnID = MatchTable[CurrentIdx++];
1041       int64_t OpIdx = MatchTable[CurrentIdx++];
1042       int64_t RendererFnID = MatchTable[CurrentIdx++];
1043       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1044 
1045       DEBUG_WITH_TYPE(
1046         TgtInstructionSelector::getName(),
1047         dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
1048                << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
1049                << OpIdx << "), "
1050         << RendererFnID << ")\n");
1051       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1052                                                      *State.MIs[OldInsnID],
1053                                                      OpIdx);
1054       break;
1055     }
1056     case GIR_ConstrainOperandRC: {
1057       int64_t InsnID = MatchTable[CurrentIdx++];
1058       int64_t OpIdx = MatchTable[CurrentIdx++];
1059       int64_t RCEnum = MatchTable[CurrentIdx++];
1060       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1061       MachineInstr &I = *OutMIs[InsnID].getInstr();
1062       MachineFunction &MF = *I.getParent()->getParent();
1063       MachineRegisterInfo &MRI = MF.getRegInfo();
1064       const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1065       MachineOperand &MO = I.getOperand(OpIdx);
1066       constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1067       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1068                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1069                              << InsnID << "], " << OpIdx << ", " << RCEnum
1070                              << ")\n");
1071       break;
1072     }
1073 
1074     case GIR_ConstrainSelectedInstOperands: {
1075       int64_t InsnID = MatchTable[CurrentIdx++];
1076       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1077       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1078                                        RBI);
1079       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1080                       dbgs() << CurrentIdx
1081                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1082                              << InsnID << "])\n");
1083       break;
1084     }
1085 
1086     case GIR_MergeMemOperands: {
1087       int64_t InsnID = MatchTable[CurrentIdx++];
1088       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1089 
1090       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1091                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1092                              << InsnID << "]");
1093       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1094       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1095              GIU_MergeMemOperands_EndOfList) {
1096         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1097                         dbgs() << ", MIs[" << MergeInsnID << "]");
1098         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1099           OutMIs[InsnID].addMemOperand(MMO);
1100       }
1101       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1102       break;
1103     }
1104 
1105     case GIR_EraseFromParent: {
1106       int64_t InsnID = MatchTable[CurrentIdx++];
1107       assert(State.MIs[InsnID] &&
1108              "Attempted to erase an undefined instruction");
1109       State.MIs[InsnID]->eraseFromParent();
1110       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1111                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1112                              << InsnID << "])\n");
1113       break;
1114     }
1115 
1116     case GIR_MakeTempReg: {
1117       int64_t TempRegID = MatchTable[CurrentIdx++];
1118       int64_t TypeID = MatchTable[CurrentIdx++];
1119 
1120       State.TempRegisters[TempRegID] =
1121           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1122       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1123                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1124                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1125       break;
1126     }
1127 
1128     case GIR_Coverage: {
1129       int64_t RuleID = MatchTable[CurrentIdx++];
1130       CoverageInfo.setCovered(RuleID);
1131 
1132       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1133                       dbgs()
1134                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1135       break;
1136     }
1137 
1138     case GIR_Done:
1139       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1140                       dbgs() << CurrentIdx << ": GIR_Done\n");
1141       propagateFlags(OutMIs);
1142       return true;
1143 
1144     default:
1145       llvm_unreachable("Unexpected command");
1146     }
1147   }
1148 }
1149 
1150 } // end namespace llvm
1151 
1152 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
1153