1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file declares the API for the instruction selector.
10 /// This class is responsible for selecting machine instructions.
11 /// It's implemented by the target. It's used by the InstructionSelect pass.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21 #include "llvm/CodeGen/GlobalISel/Utils.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <cstddef>
35 #include <cstdint>
36
37 namespace llvm {
38
39 /// GlobalISel PatFrag Predicates
40 enum {
41 GIPFP_I64_Invalid = 0,
42 GIPFP_APInt_Invalid = 0,
43 GIPFP_APFloat_Invalid = 0,
44 GIPFP_MI_Invalid = 0,
45 };
46
47 template <class TgtInstructionSelector, class PredicateBitset,
48 class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)49 bool InstructionSelector::executeMatchTable(
50 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52 &ISelInfo,
53 const int64_t *MatchTable, const TargetInstrInfo &TII,
54 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56 CodeGenCoverage &CoverageInfo) const {
57
58 uint64_t CurrentIdx = 0;
59 SmallVector<uint64_t, 4> OnFailResumeAt;
60
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 auto propagateFlags = [=](NewMIVector &OutMIs) {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 MIB.setMIFlags(MIBFlags);
87 }
88
89 return true;
90 };
91
92 while (true) {
93 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
94 int64_t MatcherOpcode = MatchTable[CurrentIdx++];
95 switch (MatcherOpcode) {
96 case GIM_Try: {
97 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
98 dbgs() << CurrentIdx << ": Begin try-block\n");
99 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
100 break;
101 }
102
103 case GIM_RecordInsn: {
104 int64_t NewInsnID = MatchTable[CurrentIdx++];
105 int64_t InsnID = MatchTable[CurrentIdx++];
106 int64_t OpIdx = MatchTable[CurrentIdx++];
107
108 // As an optimisation we require that MIs[0] is always the root. Refuse
109 // any attempt to modify it.
110 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
111
112 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
113 if (!MO.isReg()) {
114 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
115 dbgs() << CurrentIdx << ": Not a register\n");
116 if (handleReject() == RejectAndGiveUp)
117 return false;
118 break;
119 }
120 if (Register::isPhysicalRegister(MO.getReg())) {
121 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
122 dbgs() << CurrentIdx << ": Is a physical register\n");
123 if (handleReject() == RejectAndGiveUp)
124 return false;
125 break;
126 }
127
128 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
129 if ((size_t)NewInsnID < State.MIs.size())
130 State.MIs[NewInsnID] = NewMI;
131 else {
132 assert((size_t)NewInsnID == State.MIs.size() &&
133 "Expected to store MIs in order");
134 State.MIs.push_back(NewMI);
135 }
136 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
137 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
138 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
139 << ")\n");
140 break;
141 }
142
143 case GIM_CheckFeatures: {
144 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
145 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146 dbgs() << CurrentIdx
147 << ": GIM_CheckFeatures(ExpectedBitsetID="
148 << ExpectedBitsetID << ")\n");
149 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
150 ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
151 if (handleReject() == RejectAndGiveUp)
152 return false;
153 }
154 break;
155 }
156
157 case GIM_CheckOpcode:
158 case GIM_CheckOpcodeIsEither: {
159 int64_t InsnID = MatchTable[CurrentIdx++];
160 int64_t Expected0 = MatchTable[CurrentIdx++];
161 int64_t Expected1 = -1;
162 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
163 Expected1 = MatchTable[CurrentIdx++];
164
165 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
166 unsigned Opcode = State.MIs[InsnID]->getOpcode();
167
168 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
169 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
170 << "], ExpectedOpcode=" << Expected0;
171 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
172 dbgs() << " || " << Expected1;
173 dbgs() << ") // Got=" << Opcode << "\n";
174 );
175
176 if (Opcode != Expected0 && Opcode != Expected1) {
177 if (handleReject() == RejectAndGiveUp)
178 return false;
179 }
180 break;
181 }
182 case GIM_SwitchOpcode: {
183 int64_t InsnID = MatchTable[CurrentIdx++];
184 int64_t LowerBound = MatchTable[CurrentIdx++];
185 int64_t UpperBound = MatchTable[CurrentIdx++];
186 int64_t Default = MatchTable[CurrentIdx++];
187
188 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
189 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
190
191 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
192 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
193 << LowerBound << ", " << UpperBound << "), Default=" << Default
194 << ", JumpTable...) // Got=" << Opcode << "\n";
195 });
196 if (Opcode < LowerBound || UpperBound <= Opcode) {
197 CurrentIdx = Default;
198 break;
199 }
200 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
201 if (!CurrentIdx) {
202 CurrentIdx = Default;
203 break;
204 }
205 OnFailResumeAt.push_back(Default);
206 break;
207 }
208
209 case GIM_SwitchType: {
210 int64_t InsnID = MatchTable[CurrentIdx++];
211 int64_t OpIdx = MatchTable[CurrentIdx++];
212 int64_t LowerBound = MatchTable[CurrentIdx++];
213 int64_t UpperBound = MatchTable[CurrentIdx++];
214 int64_t Default = MatchTable[CurrentIdx++];
215
216 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
217 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
218
219 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
220 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
221 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
222 << UpperBound << "), Default=" << Default
223 << ", JumpTable...) // Got=";
224 if (!MO.isReg())
225 dbgs() << "Not a VReg\n";
226 else
227 dbgs() << MRI.getType(MO.getReg()) << "\n";
228 });
229 if (!MO.isReg()) {
230 CurrentIdx = Default;
231 break;
232 }
233 const LLT Ty = MRI.getType(MO.getReg());
234 const auto TyI = ISelInfo.TypeIDMap.find(Ty);
235 if (TyI == ISelInfo.TypeIDMap.end()) {
236 CurrentIdx = Default;
237 break;
238 }
239 const int64_t TypeID = TyI->second;
240 if (TypeID < LowerBound || UpperBound <= TypeID) {
241 CurrentIdx = Default;
242 break;
243 }
244 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
245 if (!CurrentIdx) {
246 CurrentIdx = Default;
247 break;
248 }
249 OnFailResumeAt.push_back(Default);
250 break;
251 }
252
253 case GIM_CheckNumOperands: {
254 int64_t InsnID = MatchTable[CurrentIdx++];
255 int64_t Expected = MatchTable[CurrentIdx++];
256 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
257 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
258 << InsnID << "], Expected=" << Expected << ")\n");
259 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
260 if (State.MIs[InsnID]->getNumOperands() != Expected) {
261 if (handleReject() == RejectAndGiveUp)
262 return false;
263 }
264 break;
265 }
266 case GIM_CheckI64ImmPredicate: {
267 int64_t InsnID = MatchTable[CurrentIdx++];
268 int64_t Predicate = MatchTable[CurrentIdx++];
269 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
270 dbgs()
271 << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
272 << InsnID << "], Predicate=" << Predicate << ")\n");
273 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
274 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
275 "Expected G_CONSTANT");
276 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
277 int64_t Value = 0;
278 if (State.MIs[InsnID]->getOperand(1).isCImm())
279 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
280 else if (State.MIs[InsnID]->getOperand(1).isImm())
281 Value = State.MIs[InsnID]->getOperand(1).getImm();
282 else
283 llvm_unreachable("Expected Imm or CImm operand");
284
285 if (!testImmPredicate_I64(Predicate, Value))
286 if (handleReject() == RejectAndGiveUp)
287 return false;
288 break;
289 }
290 case GIM_CheckAPIntImmPredicate: {
291 int64_t InsnID = MatchTable[CurrentIdx++];
292 int64_t Predicate = MatchTable[CurrentIdx++];
293 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
294 dbgs()
295 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
296 << InsnID << "], Predicate=" << Predicate << ")\n");
297 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
298 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
299 "Expected G_CONSTANT");
300 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
301 APInt Value;
302 if (State.MIs[InsnID]->getOperand(1).isCImm())
303 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
304 else
305 llvm_unreachable("Expected Imm or CImm operand");
306
307 if (!testImmPredicate_APInt(Predicate, Value))
308 if (handleReject() == RejectAndGiveUp)
309 return false;
310 break;
311 }
312 case GIM_CheckAPFloatImmPredicate: {
313 int64_t InsnID = MatchTable[CurrentIdx++];
314 int64_t Predicate = MatchTable[CurrentIdx++];
315 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
316 dbgs()
317 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
318 << InsnID << "], Predicate=" << Predicate << ")\n");
319 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
320 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
321 "Expected G_FCONSTANT");
322 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
323 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
324 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
325
326 if (!testImmPredicate_APFloat(Predicate, Value))
327 if (handleReject() == RejectAndGiveUp)
328 return false;
329 break;
330 }
331 case GIM_CheckIsBuildVectorAllOnes:
332 case GIM_CheckIsBuildVectorAllZeros: {
333 int64_t InsnID = MatchTable[CurrentIdx++];
334
335 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
336 dbgs() << CurrentIdx
337 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
338 << InsnID << "])\n");
339 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
340
341 const MachineInstr *MI = State.MIs[InsnID];
342 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
343 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
344 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
345
346 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
347 if (!isBuildVectorAllOnes(*MI, MRI)) {
348 if (handleReject() == RejectAndGiveUp)
349 return false;
350 }
351 } else {
352 if (!isBuildVectorAllZeros(*MI, MRI)) {
353 if (handleReject() == RejectAndGiveUp)
354 return false;
355 }
356 }
357
358 break;
359 }
360 case GIM_CheckCxxInsnPredicate: {
361 int64_t InsnID = MatchTable[CurrentIdx++];
362 int64_t Predicate = MatchTable[CurrentIdx++];
363 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
364 dbgs()
365 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
366 << InsnID << "], Predicate=" << Predicate << ")\n");
367 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
368 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
369
370 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
371 State.RecordedOperands))
372 if (handleReject() == RejectAndGiveUp)
373 return false;
374 break;
375 }
376 case GIM_CheckAtomicOrdering: {
377 int64_t InsnID = MatchTable[CurrentIdx++];
378 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
379 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
381 << InsnID << "], " << (uint64_t)Ordering << ")\n");
382 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
383 if (!State.MIs[InsnID]->hasOneMemOperand())
384 if (handleReject() == RejectAndGiveUp)
385 return false;
386
387 for (const auto &MMO : State.MIs[InsnID]->memoperands())
388 if (MMO->getOrdering() != Ordering)
389 if (handleReject() == RejectAndGiveUp)
390 return false;
391 break;
392 }
393 case GIM_CheckAtomicOrderingOrStrongerThan: {
394 int64_t InsnID = MatchTable[CurrentIdx++];
395 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
396 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
397 dbgs() << CurrentIdx
398 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
399 << InsnID << "], " << (uint64_t)Ordering << ")\n");
400 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
401 if (!State.MIs[InsnID]->hasOneMemOperand())
402 if (handleReject() == RejectAndGiveUp)
403 return false;
404
405 for (const auto &MMO : State.MIs[InsnID]->memoperands())
406 if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
407 if (handleReject() == RejectAndGiveUp)
408 return false;
409 break;
410 }
411 case GIM_CheckAtomicOrderingWeakerThan: {
412 int64_t InsnID = MatchTable[CurrentIdx++];
413 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
414 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
415 dbgs() << CurrentIdx
416 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
417 << InsnID << "], " << (uint64_t)Ordering << ")\n");
418 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
419 if (!State.MIs[InsnID]->hasOneMemOperand())
420 if (handleReject() == RejectAndGiveUp)
421 return false;
422
423 for (const auto &MMO : State.MIs[InsnID]->memoperands())
424 if (!isStrongerThan(Ordering, MMO->getOrdering()))
425 if (handleReject() == RejectAndGiveUp)
426 return false;
427 break;
428 }
429 case GIM_CheckMemoryAddressSpace: {
430 int64_t InsnID = MatchTable[CurrentIdx++];
431 int64_t MMOIdx = MatchTable[CurrentIdx++];
432 // This accepts a list of possible address spaces.
433 const int NumAddrSpace = MatchTable[CurrentIdx++];
434
435 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
436 if (handleReject() == RejectAndGiveUp)
437 return false;
438 break;
439 }
440
441 // Need to still jump to the end of the list of address spaces if we find
442 // a match earlier.
443 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
444
445 const MachineMemOperand *MMO
446 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
447 const unsigned MMOAddrSpace = MMO->getAddrSpace();
448
449 bool Success = false;
450 for (int I = 0; I != NumAddrSpace; ++I) {
451 unsigned AddrSpace = MatchTable[CurrentIdx++];
452 DEBUG_WITH_TYPE(
453 TgtInstructionSelector::getName(),
454 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
455 << AddrSpace << '\n');
456
457 if (AddrSpace == MMOAddrSpace) {
458 Success = true;
459 break;
460 }
461 }
462
463 CurrentIdx = LastIdx;
464 if (!Success && handleReject() == RejectAndGiveUp)
465 return false;
466 break;
467 }
468 case GIM_CheckMemoryAlignment: {
469 int64_t InsnID = MatchTable[CurrentIdx++];
470 int64_t MMOIdx = MatchTable[CurrentIdx++];
471 unsigned MinAlign = MatchTable[CurrentIdx++];
472
473 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
474
475 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
476 if (handleReject() == RejectAndGiveUp)
477 return false;
478 break;
479 }
480
481 MachineMemOperand *MMO
482 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
483 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
484 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
485 << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
486 << ")->getAlignment() >= " << MinAlign << ")\n");
487 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
488 return false;
489
490 break;
491 }
492 case GIM_CheckMemorySizeEqualTo: {
493 int64_t InsnID = MatchTable[CurrentIdx++];
494 int64_t MMOIdx = MatchTable[CurrentIdx++];
495 uint64_t Size = MatchTable[CurrentIdx++];
496
497 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498 dbgs() << CurrentIdx
499 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
500 << "]->memoperands() + " << MMOIdx
501 << ", Size=" << Size << ")\n");
502 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
503
504 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
505 if (handleReject() == RejectAndGiveUp)
506 return false;
507 break;
508 }
509
510 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
511
512 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
513 dbgs() << MMO->getSize() << " bytes vs " << Size
514 << " bytes\n");
515 if (MMO->getSize() != Size)
516 if (handleReject() == RejectAndGiveUp)
517 return false;
518
519 break;
520 }
521 case GIM_CheckMemorySizeEqualToLLT:
522 case GIM_CheckMemorySizeLessThanLLT:
523 case GIM_CheckMemorySizeGreaterThanLLT: {
524 int64_t InsnID = MatchTable[CurrentIdx++];
525 int64_t MMOIdx = MatchTable[CurrentIdx++];
526 int64_t OpIdx = MatchTable[CurrentIdx++];
527
528 DEBUG_WITH_TYPE(
529 TgtInstructionSelector::getName(),
530 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
531 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
532 ? "EqualTo"
533 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
534 ? "GreaterThan"
535 : "LessThan")
536 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
537 << ", OpIdx=" << OpIdx << ")\n");
538 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
539
540 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
541 if (!MO.isReg()) {
542 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
543 dbgs() << CurrentIdx << ": Not a register\n");
544 if (handleReject() == RejectAndGiveUp)
545 return false;
546 break;
547 }
548
549 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
550 if (handleReject() == RejectAndGiveUp)
551 return false;
552 break;
553 }
554
555 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
556
557 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
558 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
559 MMO->getSizeInBits() != Size) {
560 if (handleReject() == RejectAndGiveUp)
561 return false;
562 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
563 MMO->getSizeInBits() >= Size) {
564 if (handleReject() == RejectAndGiveUp)
565 return false;
566 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
567 MMO->getSizeInBits() <= Size)
568 if (handleReject() == RejectAndGiveUp)
569 return false;
570
571 break;
572 }
573 case GIM_CheckType: {
574 int64_t InsnID = MatchTable[CurrentIdx++];
575 int64_t OpIdx = MatchTable[CurrentIdx++];
576 int64_t TypeID = MatchTable[CurrentIdx++];
577 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
578 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
579 << "]->getOperand(" << OpIdx
580 << "), TypeID=" << TypeID << ")\n");
581 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
582 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
583 if (!MO.isReg() ||
584 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
585 if (handleReject() == RejectAndGiveUp)
586 return false;
587 }
588 break;
589 }
590 case GIM_CheckPointerToAny: {
591 int64_t InsnID = MatchTable[CurrentIdx++];
592 int64_t OpIdx = MatchTable[CurrentIdx++];
593 int64_t SizeInBits = MatchTable[CurrentIdx++];
594
595 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
596 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
597 << InsnID << "]->getOperand(" << OpIdx
598 << "), SizeInBits=" << SizeInBits << ")\n");
599 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
600 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
601 const LLT Ty = MRI.getType(MO.getReg());
602
603 // iPTR must be looked up in the target.
604 if (SizeInBits == 0) {
605 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
606 const unsigned AddrSpace = Ty.getAddressSpace();
607 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
608 }
609
610 assert(SizeInBits != 0 && "Pointer size must be known");
611
612 if (MO.isReg()) {
613 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
614 if (handleReject() == RejectAndGiveUp)
615 return false;
616 } else if (handleReject() == RejectAndGiveUp)
617 return false;
618
619 break;
620 }
621 case GIM_RecordNamedOperand: {
622 int64_t InsnID = MatchTable[CurrentIdx++];
623 int64_t OpIdx = MatchTable[CurrentIdx++];
624 uint64_t StoreIdx = MatchTable[CurrentIdx++];
625
626 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
627 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
628 << InsnID << "]->getOperand(" << OpIdx
629 << "), StoreIdx=" << StoreIdx << ")\n");
630 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
631 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
632 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
633 break;
634 }
635 case GIM_CheckRegBankForClass: {
636 int64_t InsnID = MatchTable[CurrentIdx++];
637 int64_t OpIdx = MatchTable[CurrentIdx++];
638 int64_t RCEnum = MatchTable[CurrentIdx++];
639 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
640 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
641 << InsnID << "]->getOperand(" << OpIdx
642 << "), RCEnum=" << RCEnum << ")\n");
643 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
644 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
645 if (!MO.isReg() ||
646 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
647 MRI.getType(MO.getReg())) !=
648 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
649 if (handleReject() == RejectAndGiveUp)
650 return false;
651 }
652 break;
653 }
654
655 case GIM_CheckComplexPattern: {
656 int64_t InsnID = MatchTable[CurrentIdx++];
657 int64_t OpIdx = MatchTable[CurrentIdx++];
658 int64_t RendererID = MatchTable[CurrentIdx++];
659 int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
660 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
661 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
662 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
663 << "]->getOperand(" << OpIdx
664 << "), ComplexPredicateID=" << ComplexPredicateID
665 << ")\n");
666 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
667 // FIXME: Use std::invoke() when it's available.
668 ComplexRendererFns Renderer =
669 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
670 State.MIs[InsnID]->getOperand(OpIdx));
671 if (Renderer.hasValue())
672 State.Renderers[RendererID] = Renderer.getValue();
673 else
674 if (handleReject() == RejectAndGiveUp)
675 return false;
676 break;
677 }
678
679 case GIM_CheckConstantInt: {
680 int64_t InsnID = MatchTable[CurrentIdx++];
681 int64_t OpIdx = MatchTable[CurrentIdx++];
682 int64_t Value = MatchTable[CurrentIdx++];
683 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
684 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
685 << InsnID << "]->getOperand(" << OpIdx
686 << "), Value=" << Value << ")\n");
687 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
688 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
689 if (MO.isReg()) {
690 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
691 LLT Ty = MRI.getType(MO.getReg());
692 Value = SignExtend64(Value, Ty.getSizeInBits());
693
694 if (!isOperandImmEqual(MO, Value, MRI)) {
695 if (handleReject() == RejectAndGiveUp)
696 return false;
697 }
698 } else if (handleReject() == RejectAndGiveUp)
699 return false;
700
701 break;
702 }
703
704 case GIM_CheckLiteralInt: {
705 int64_t InsnID = MatchTable[CurrentIdx++];
706 int64_t OpIdx = MatchTable[CurrentIdx++];
707 int64_t Value = MatchTable[CurrentIdx++];
708 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
710 << InsnID << "]->getOperand(" << OpIdx
711 << "), Value=" << Value << ")\n");
712 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
713 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
714 if (MO.isImm() && MO.getImm() == Value)
715 break;
716
717 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
718 break;
719
720 if (handleReject() == RejectAndGiveUp)
721 return false;
722
723 break;
724 }
725
726 case GIM_CheckIntrinsicID: {
727 int64_t InsnID = MatchTable[CurrentIdx++];
728 int64_t OpIdx = MatchTable[CurrentIdx++];
729 int64_t Value = MatchTable[CurrentIdx++];
730 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
732 << InsnID << "]->getOperand(" << OpIdx
733 << "), Value=" << Value << ")\n");
734 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
735 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
736 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
737 if (handleReject() == RejectAndGiveUp)
738 return false;
739 break;
740 }
741 case GIM_CheckCmpPredicate: {
742 int64_t InsnID = MatchTable[CurrentIdx++];
743 int64_t OpIdx = MatchTable[CurrentIdx++];
744 int64_t Value = MatchTable[CurrentIdx++];
745 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
747 << InsnID << "]->getOperand(" << OpIdx
748 << "), Value=" << Value << ")\n");
749 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
750 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
751 if (!MO.isPredicate() || MO.getPredicate() != Value)
752 if (handleReject() == RejectAndGiveUp)
753 return false;
754 break;
755 }
756 case GIM_CheckIsMBB: {
757 int64_t InsnID = MatchTable[CurrentIdx++];
758 int64_t OpIdx = MatchTable[CurrentIdx++];
759 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
760 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
761 << "]->getOperand(" << OpIdx << "))\n");
762 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
763 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
764 if (handleReject() == RejectAndGiveUp)
765 return false;
766 }
767 break;
768 }
769 case GIM_CheckIsImm: {
770 int64_t InsnID = MatchTable[CurrentIdx++];
771 int64_t OpIdx = MatchTable[CurrentIdx++];
772 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
773 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
774 << "]->getOperand(" << OpIdx << "))\n");
775 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
776 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
777 if (handleReject() == RejectAndGiveUp)
778 return false;
779 }
780 break;
781 }
782 case GIM_CheckIsSafeToFold: {
783 int64_t InsnID = MatchTable[CurrentIdx++];
784 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
785 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
786 << InsnID << "])\n");
787 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
788 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
789 if (handleReject() == RejectAndGiveUp)
790 return false;
791 }
792 break;
793 }
794 case GIM_CheckIsSameOperand: {
795 int64_t InsnID = MatchTable[CurrentIdx++];
796 int64_t OpIdx = MatchTable[CurrentIdx++];
797 int64_t OtherInsnID = MatchTable[CurrentIdx++];
798 int64_t OtherOpIdx = MatchTable[CurrentIdx++];
799 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
800 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
801 << InsnID << "][" << OpIdx << "], MIs["
802 << OtherInsnID << "][" << OtherOpIdx << "])\n");
803 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
804 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
805 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
806 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
807 if (handleReject() == RejectAndGiveUp)
808 return false;
809 }
810 break;
811 }
812 case GIM_Reject:
813 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
814 dbgs() << CurrentIdx << ": GIM_Reject\n");
815 if (handleReject() == RejectAndGiveUp)
816 return false;
817 break;
818
819 case GIR_MutateOpcode: {
820 int64_t OldInsnID = MatchTable[CurrentIdx++];
821 uint64_t NewInsnID = MatchTable[CurrentIdx++];
822 int64_t NewOpcode = MatchTable[CurrentIdx++];
823 if (NewInsnID >= OutMIs.size())
824 OutMIs.resize(NewInsnID + 1);
825
826 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
827 State.MIs[OldInsnID]);
828 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
829 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
830 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
831 << NewInsnID << "], MIs[" << OldInsnID << "], "
832 << NewOpcode << ")\n");
833 break;
834 }
835
836 case GIR_BuildMI: {
837 uint64_t NewInsnID = MatchTable[CurrentIdx++];
838 int64_t Opcode = MatchTable[CurrentIdx++];
839 if (NewInsnID >= OutMIs.size())
840 OutMIs.resize(NewInsnID + 1);
841
842 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
843 State.MIs[0]->getDebugLoc(), TII.get(Opcode));
844 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
846 << NewInsnID << "], " << Opcode << ")\n");
847 break;
848 }
849
850 case GIR_Copy: {
851 int64_t NewInsnID = MatchTable[CurrentIdx++];
852 int64_t OldInsnID = MatchTable[CurrentIdx++];
853 int64_t OpIdx = MatchTable[CurrentIdx++];
854 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
855 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
856 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
857 dbgs()
858 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
859 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
860 break;
861 }
862
863 case GIR_CopyOrAddZeroReg: {
864 int64_t NewInsnID = MatchTable[CurrentIdx++];
865 int64_t OldInsnID = MatchTable[CurrentIdx++];
866 int64_t OpIdx = MatchTable[CurrentIdx++];
867 int64_t ZeroReg = MatchTable[CurrentIdx++];
868 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
869 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
870 if (isOperandImmEqual(MO, 0, MRI))
871 OutMIs[NewInsnID].addReg(ZeroReg);
872 else
873 OutMIs[NewInsnID].add(MO);
874 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
875 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
876 << NewInsnID << "], MIs[" << OldInsnID << "], "
877 << OpIdx << ", " << ZeroReg << ")\n");
878 break;
879 }
880
881 case GIR_CopySubReg: {
882 int64_t NewInsnID = MatchTable[CurrentIdx++];
883 int64_t OldInsnID = MatchTable[CurrentIdx++];
884 int64_t OpIdx = MatchTable[CurrentIdx++];
885 int64_t SubRegIdx = MatchTable[CurrentIdx++];
886 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
887 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
888 0, SubRegIdx);
889 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
891 << NewInsnID << "], MIs[" << OldInsnID << "], "
892 << OpIdx << ", " << SubRegIdx << ")\n");
893 break;
894 }
895
896 case GIR_AddImplicitDef: {
897 int64_t InsnID = MatchTable[CurrentIdx++];
898 int64_t RegNum = MatchTable[CurrentIdx++];
899 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
900 OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
901 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
902 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
903 << InsnID << "], " << RegNum << ")\n");
904 break;
905 }
906
907 case GIR_AddImplicitUse: {
908 int64_t InsnID = MatchTable[CurrentIdx++];
909 int64_t RegNum = MatchTable[CurrentIdx++];
910 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
911 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
912 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
913 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
914 << InsnID << "], " << RegNum << ")\n");
915 break;
916 }
917
918 case GIR_AddRegister: {
919 int64_t InsnID = MatchTable[CurrentIdx++];
920 int64_t RegNum = MatchTable[CurrentIdx++];
921 uint64_t RegFlags = MatchTable[CurrentIdx++];
922 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
923 OutMIs[InsnID].addReg(RegNum, RegFlags);
924 DEBUG_WITH_TYPE(
925 TgtInstructionSelector::getName(),
926 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
927 << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
928 break;
929 }
930
931 case GIR_AddTempRegister:
932 case GIR_AddTempSubRegister: {
933 int64_t InsnID = MatchTable[CurrentIdx++];
934 int64_t TempRegID = MatchTable[CurrentIdx++];
935 uint64_t TempRegFlags = MatchTable[CurrentIdx++];
936 unsigned SubReg = 0;
937 if (MatcherOpcode == GIR_AddTempSubRegister)
938 SubReg = MatchTable[CurrentIdx++];
939
940 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
941
942 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
943 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
944 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
945 << InsnID << "], TempRegisters[" << TempRegID
946 << "]";
947 if (SubReg)
948 dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
949 dbgs() << ", " << TempRegFlags << ")\n");
950 break;
951 }
952
953 case GIR_AddImm: {
954 int64_t InsnID = MatchTable[CurrentIdx++];
955 int64_t Imm = MatchTable[CurrentIdx++];
956 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
957 OutMIs[InsnID].addImm(Imm);
958 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
959 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
960 << "], " << Imm << ")\n");
961 break;
962 }
963
964 case GIR_ComplexRenderer: {
965 int64_t InsnID = MatchTable[CurrentIdx++];
966 int64_t RendererID = MatchTable[CurrentIdx++];
967 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
968 for (const auto &RenderOpFn : State.Renderers[RendererID])
969 RenderOpFn(OutMIs[InsnID]);
970 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
971 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
972 << InsnID << "], " << RendererID << ")\n");
973 break;
974 }
975 case GIR_ComplexSubOperandRenderer: {
976 int64_t InsnID = MatchTable[CurrentIdx++];
977 int64_t RendererID = MatchTable[CurrentIdx++];
978 int64_t RenderOpID = MatchTable[CurrentIdx++];
979 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
980 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
981 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
982 dbgs() << CurrentIdx
983 << ": GIR_ComplexSubOperandRenderer(OutMIs["
984 << InsnID << "], " << RendererID << ", "
985 << RenderOpID << ")\n");
986 break;
987 }
988
989 case GIR_CopyConstantAsSImm: {
990 int64_t NewInsnID = MatchTable[CurrentIdx++];
991 int64_t OldInsnID = MatchTable[CurrentIdx++];
992 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
993 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
994 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
995 OutMIs[NewInsnID].addImm(
996 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
997 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
998 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
999 else
1000 llvm_unreachable("Expected Imm or CImm operand");
1001 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1002 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1003 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1004 break;
1005 }
1006
1007 // TODO: Needs a test case once we have a pattern that uses this.
1008 case GIR_CopyFConstantAsFPImm: {
1009 int64_t NewInsnID = MatchTable[CurrentIdx++];
1010 int64_t OldInsnID = MatchTable[CurrentIdx++];
1011 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1012 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
1013 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1014 OutMIs[NewInsnID].addFPImm(
1015 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1016 else
1017 llvm_unreachable("Expected FPImm operand");
1018 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1019 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1020 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1021 break;
1022 }
1023
1024 case GIR_CustomRenderer: {
1025 int64_t InsnID = MatchTable[CurrentIdx++];
1026 int64_t OldInsnID = MatchTable[CurrentIdx++];
1027 int64_t RendererFnID = MatchTable[CurrentIdx++];
1028 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1029 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1030 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1031 << InsnID << "], MIs[" << OldInsnID << "], "
1032 << RendererFnID << ")\n");
1033 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
1034 OutMIs[InsnID], *State.MIs[OldInsnID],
1035 -1); // Not a source operand of the old instruction.
1036 break;
1037 }
1038 case GIR_CustomOperandRenderer: {
1039 int64_t InsnID = MatchTable[CurrentIdx++];
1040 int64_t OldInsnID = MatchTable[CurrentIdx++];
1041 int64_t OpIdx = MatchTable[CurrentIdx++];
1042 int64_t RendererFnID = MatchTable[CurrentIdx++];
1043 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1044
1045 DEBUG_WITH_TYPE(
1046 TgtInstructionSelector::getName(),
1047 dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
1048 << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
1049 << OpIdx << "), "
1050 << RendererFnID << ")\n");
1051 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1052 *State.MIs[OldInsnID],
1053 OpIdx);
1054 break;
1055 }
1056 case GIR_ConstrainOperandRC: {
1057 int64_t InsnID = MatchTable[CurrentIdx++];
1058 int64_t OpIdx = MatchTable[CurrentIdx++];
1059 int64_t RCEnum = MatchTable[CurrentIdx++];
1060 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1061 MachineInstr &I = *OutMIs[InsnID].getInstr();
1062 MachineFunction &MF = *I.getParent()->getParent();
1063 MachineRegisterInfo &MRI = MF.getRegInfo();
1064 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1065 MachineOperand &MO = I.getOperand(OpIdx);
1066 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1067 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1068 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1069 << InsnID << "], " << OpIdx << ", " << RCEnum
1070 << ")\n");
1071 break;
1072 }
1073
1074 case GIR_ConstrainSelectedInstOperands: {
1075 int64_t InsnID = MatchTable[CurrentIdx++];
1076 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1077 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1078 RBI);
1079 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1080 dbgs() << CurrentIdx
1081 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1082 << InsnID << "])\n");
1083 break;
1084 }
1085
1086 case GIR_MergeMemOperands: {
1087 int64_t InsnID = MatchTable[CurrentIdx++];
1088 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1089
1090 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1091 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1092 << InsnID << "]");
1093 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1094 while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1095 GIU_MergeMemOperands_EndOfList) {
1096 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1097 dbgs() << ", MIs[" << MergeInsnID << "]");
1098 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1099 OutMIs[InsnID].addMemOperand(MMO);
1100 }
1101 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1102 break;
1103 }
1104
1105 case GIR_EraseFromParent: {
1106 int64_t InsnID = MatchTable[CurrentIdx++];
1107 assert(State.MIs[InsnID] &&
1108 "Attempted to erase an undefined instruction");
1109 State.MIs[InsnID]->eraseFromParent();
1110 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1111 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1112 << InsnID << "])\n");
1113 break;
1114 }
1115
1116 case GIR_MakeTempReg: {
1117 int64_t TempRegID = MatchTable[CurrentIdx++];
1118 int64_t TypeID = MatchTable[CurrentIdx++];
1119
1120 State.TempRegisters[TempRegID] =
1121 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1122 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1123 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1124 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1125 break;
1126 }
1127
1128 case GIR_Coverage: {
1129 int64_t RuleID = MatchTable[CurrentIdx++];
1130 CoverageInfo.setCovered(RuleID);
1131
1132 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1133 dbgs()
1134 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1135 break;
1136 }
1137
1138 case GIR_Done:
1139 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1140 dbgs() << CurrentIdx << ": GIR_Done\n");
1141 propagateFlags(OutMIs);
1142 return true;
1143
1144 default:
1145 llvm_unreachable("Unexpected command");
1146 }
1147 }
1148 }
1149
1150 } // end namespace llvm
1151
1152 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
1153