1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable` 10 /// function. This is implemented in a separate file because the function is 11 /// quite large. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 17 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" 20 #include "llvm/CodeGen/GlobalISel/Utils.h" 21 #include "llvm/CodeGen/MachineInstrBuilder.h" 22 #include "llvm/CodeGen/MachineOperand.h" 23 #include "llvm/CodeGen/MachineRegisterInfo.h" 24 #include "llvm/CodeGen/RegisterBankInfo.h" 25 #include "llvm/CodeGen/TargetInstrInfo.h" 26 #include "llvm/CodeGen/TargetOpcodes.h" 27 #include "llvm/CodeGen/TargetRegisterInfo.h" 28 #include "llvm/IR/Constants.h" 29 #include "llvm/IR/DataLayout.h" 30 #include "llvm/Support/CodeGenCoverage.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Support/ErrorHandling.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <cassert> 35 #include <cstddef> 36 #include <cstdint> 37 38 namespace llvm { 39 40 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, 41 class CustomRendererFn> 42 bool GIMatchTableExecutor::executeMatchTable( 43 TgtExecutor &Exec, NewMIVector &OutMIs, MatcherState &State, 44 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> 45 &ExecInfo, 46 const int64_t *MatchTable, const TargetInstrInfo &TII, 47 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, 48 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, 49 CodeGenCoverage *CoverageInfo) const { 50 51 uint64_t CurrentIdx = 0; 52 SmallVector<uint64_t, 4> OnFailResumeAt; 53 54 // Bypass the flag check on the instruction, and only look at the MCInstrDesc. 55 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); 56 57 const uint16_t Flags = State.MIs[0]->getFlags(); 58 59 enum RejectAction { RejectAndGiveUp, RejectAndResume }; 60 auto handleReject = [&]() -> RejectAction { 61 DEBUG_WITH_TYPE(TgtExecutor::getName(), 62 dbgs() << CurrentIdx << ": Rejected\n"); 63 if (OnFailResumeAt.empty()) 64 return RejectAndGiveUp; 65 CurrentIdx = OnFailResumeAt.pop_back_val(); 66 DEBUG_WITH_TYPE(TgtExecutor::getName(), 67 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" 68 << OnFailResumeAt.size() << " try-blocks remain)\n"); 69 return RejectAndResume; 70 }; 71 72 auto propagateFlags = [=](NewMIVector &OutMIs) { 73 for (auto MIB : OutMIs) { 74 // Set the NoFPExcept flag when no original matched instruction could 75 // raise an FP exception, but the new instruction potentially might. 76 uint16_t MIBFlags = Flags; 77 if (NoFPException && MIB->mayRaiseFPException()) 78 MIBFlags |= MachineInstr::NoFPExcept; 79 MIB.setMIFlags(MIBFlags); 80 } 81 82 return true; 83 }; 84 85 while (true) { 86 assert(CurrentIdx != ~0u && "Invalid MatchTable index"); 87 int64_t MatcherOpcode = MatchTable[CurrentIdx++]; 88 switch (MatcherOpcode) { 89 case GIM_Try: { 90 DEBUG_WITH_TYPE(TgtExecutor::getName(), 91 dbgs() << CurrentIdx << ": Begin try-block\n"); 92 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); 93 break; 94 } 95 96 case GIM_RecordInsn: 97 case GIM_RecordInsnIgnoreCopies: { 98 int64_t NewInsnID = MatchTable[CurrentIdx++]; 99 int64_t InsnID = MatchTable[CurrentIdx++]; 100 int64_t OpIdx = MatchTable[CurrentIdx++]; 101 102 // As an optimisation we require that MIs[0] is always the root. Refuse 103 // any attempt to modify it. 104 assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); 105 106 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 107 if (!MO.isReg()) { 108 DEBUG_WITH_TYPE(TgtExecutor::getName(), 109 dbgs() << CurrentIdx << ": Not a register\n"); 110 if (handleReject() == RejectAndGiveUp) 111 return false; 112 break; 113 } 114 if (MO.getReg().isPhysical()) { 115 DEBUG_WITH_TYPE(TgtExecutor::getName(), 116 dbgs() << CurrentIdx << ": Is a physical register\n"); 117 if (handleReject() == RejectAndGiveUp) 118 return false; 119 break; 120 } 121 122 MachineInstr *NewMI; 123 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) 124 NewMI = getDefIgnoringCopies(MO.getReg(), MRI); 125 else 126 NewMI = MRI.getVRegDef(MO.getReg()); 127 128 if ((size_t)NewInsnID < State.MIs.size()) 129 State.MIs[NewInsnID] = NewMI; 130 else { 131 assert((size_t)NewInsnID == State.MIs.size() && 132 "Expected to store MIs in order"); 133 State.MIs.push_back(NewMI); 134 } 135 DEBUG_WITH_TYPE(TgtExecutor::getName(), 136 dbgs() << CurrentIdx << ": MIs[" << NewInsnID 137 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx 138 << ")\n"); 139 break; 140 } 141 142 case GIM_CheckFeatures: { 143 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; 144 DEBUG_WITH_TYPE(TgtExecutor::getName(), 145 dbgs() << CurrentIdx 146 << ": GIM_CheckFeatures(ExpectedBitsetID=" 147 << ExpectedBitsetID << ")\n"); 148 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != 149 ExecInfo.FeatureBitsets[ExpectedBitsetID]) { 150 if (handleReject() == RejectAndGiveUp) 151 return false; 152 } 153 break; 154 } 155 case GIM_CheckOpcode: 156 case GIM_CheckOpcodeIsEither: { 157 int64_t InsnID = MatchTable[CurrentIdx++]; 158 int64_t Expected0 = MatchTable[CurrentIdx++]; 159 int64_t Expected1 = -1; 160 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 161 Expected1 = MatchTable[CurrentIdx++]; 162 163 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 164 unsigned Opcode = State.MIs[InsnID]->getOpcode(); 165 166 DEBUG_WITH_TYPE(TgtExecutor::getName(), 167 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID 168 << "], ExpectedOpcode=" << Expected0; 169 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() 170 << " || " << Expected1; 171 dbgs() << ") // Got=" << Opcode << "\n";); 172 173 if (Opcode != Expected0 && Opcode != Expected1) { 174 if (handleReject() == RejectAndGiveUp) 175 return false; 176 } 177 break; 178 } 179 case GIM_SwitchOpcode: { 180 int64_t InsnID = MatchTable[CurrentIdx++]; 181 int64_t LowerBound = MatchTable[CurrentIdx++]; 182 int64_t UpperBound = MatchTable[CurrentIdx++]; 183 int64_t Default = MatchTable[CurrentIdx++]; 184 185 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 186 const int64_t Opcode = State.MIs[InsnID]->getOpcode(); 187 188 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 189 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" 190 << LowerBound << ", " << UpperBound << "), Default=" << Default 191 << ", JumpTable...) // Got=" << Opcode << "\n"; 192 }); 193 if (Opcode < LowerBound || UpperBound <= Opcode) { 194 CurrentIdx = Default; 195 break; 196 } 197 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; 198 if (!CurrentIdx) { 199 CurrentIdx = Default; 200 break; 201 } 202 OnFailResumeAt.push_back(Default); 203 break; 204 } 205 206 case GIM_SwitchType: { 207 int64_t InsnID = MatchTable[CurrentIdx++]; 208 int64_t OpIdx = MatchTable[CurrentIdx++]; 209 int64_t LowerBound = MatchTable[CurrentIdx++]; 210 int64_t UpperBound = MatchTable[CurrentIdx++]; 211 int64_t Default = MatchTable[CurrentIdx++]; 212 213 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 214 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 215 216 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 217 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID 218 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " 219 << UpperBound << "), Default=" << Default 220 << ", JumpTable...) // Got="; 221 if (!MO.isReg()) 222 dbgs() << "Not a VReg\n"; 223 else 224 dbgs() << MRI.getType(MO.getReg()) << "\n"; 225 }); 226 if (!MO.isReg()) { 227 CurrentIdx = Default; 228 break; 229 } 230 const LLT Ty = MRI.getType(MO.getReg()); 231 const auto TyI = ExecInfo.TypeIDMap.find(Ty); 232 if (TyI == ExecInfo.TypeIDMap.end()) { 233 CurrentIdx = Default; 234 break; 235 } 236 const int64_t TypeID = TyI->second; 237 if (TypeID < LowerBound || UpperBound <= TypeID) { 238 CurrentIdx = Default; 239 break; 240 } 241 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; 242 if (!CurrentIdx) { 243 CurrentIdx = Default; 244 break; 245 } 246 OnFailResumeAt.push_back(Default); 247 break; 248 } 249 250 case GIM_CheckNumOperands: { 251 int64_t InsnID = MatchTable[CurrentIdx++]; 252 int64_t Expected = MatchTable[CurrentIdx++]; 253 DEBUG_WITH_TYPE(TgtExecutor::getName(), 254 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" 255 << InsnID << "], Expected=" << Expected << ")\n"); 256 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 257 if (State.MIs[InsnID]->getNumOperands() != Expected) { 258 if (handleReject() == RejectAndGiveUp) 259 return false; 260 } 261 break; 262 } 263 case GIM_CheckI64ImmPredicate: 264 case GIM_CheckImmOperandPredicate: { 265 int64_t InsnID = MatchTable[CurrentIdx++]; 266 int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate 267 ? MatchTable[CurrentIdx++] 268 : 1; 269 int64_t Predicate = MatchTable[CurrentIdx++]; 270 DEBUG_WITH_TYPE(TgtExecutor::getName(), 271 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" 272 << InsnID << "]->getOperand(" << OpIdx 273 << "), Predicate=" << Predicate << ")\n"); 274 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 275 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || 276 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && 277 "Expected immediate operand"); 278 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 279 int64_t Value = 0; 280 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) 281 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); 282 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) 283 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); 284 else 285 llvm_unreachable("Expected Imm or CImm operand"); 286 287 if (!testImmPredicate_I64(Predicate, Value)) 288 if (handleReject() == RejectAndGiveUp) 289 return false; 290 break; 291 } 292 case GIM_CheckAPIntImmPredicate: { 293 int64_t InsnID = MatchTable[CurrentIdx++]; 294 int64_t Predicate = MatchTable[CurrentIdx++]; 295 DEBUG_WITH_TYPE(TgtExecutor::getName(), 296 dbgs() 297 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" 298 << InsnID << "], Predicate=" << Predicate << ")\n"); 299 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 300 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 301 "Expected G_CONSTANT"); 302 assert(Predicate > GICXXPred_Invalid && 303 "Expected a valid predicate"); 304 if (!State.MIs[InsnID]->getOperand(1).isCImm()) 305 llvm_unreachable("Expected Imm or CImm operand"); 306 307 const APInt &Value = 308 State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); 309 if (!testImmPredicate_APInt(Predicate, Value)) 310 if (handleReject() == RejectAndGiveUp) 311 return false; 312 break; 313 } 314 case GIM_CheckAPFloatImmPredicate: { 315 int64_t InsnID = MatchTable[CurrentIdx++]; 316 int64_t Predicate = MatchTable[CurrentIdx++]; 317 DEBUG_WITH_TYPE(TgtExecutor::getName(), 318 dbgs() 319 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" 320 << InsnID << "], Predicate=" << Predicate << ")\n"); 321 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 322 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 323 "Expected G_FCONSTANT"); 324 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && 325 "Expected FPImm operand"); 326 assert(Predicate > GICXXPred_Invalid && 327 "Expected a valid predicate"); 328 const APFloat &Value = 329 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); 330 331 if (!testImmPredicate_APFloat(Predicate, Value)) 332 if (handleReject() == RejectAndGiveUp) 333 return false; 334 break; 335 } 336 case GIM_CheckIsBuildVectorAllOnes: 337 case GIM_CheckIsBuildVectorAllZeros: { 338 int64_t InsnID = MatchTable[CurrentIdx++]; 339 340 DEBUG_WITH_TYPE(TgtExecutor::getName(), 341 dbgs() << CurrentIdx 342 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" 343 << InsnID << "])\n"); 344 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 345 346 const MachineInstr *MI = State.MIs[InsnID]; 347 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || 348 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && 349 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); 350 351 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { 352 if (!isBuildVectorAllOnes(*MI, MRI)) { 353 if (handleReject() == RejectAndGiveUp) 354 return false; 355 } 356 } else { 357 if (!isBuildVectorAllZeros(*MI, MRI)) { 358 if (handleReject() == RejectAndGiveUp) 359 return false; 360 } 361 } 362 363 break; 364 } 365 case GIM_CheckSimplePredicate: { 366 // Note: we don't check for invalid here because this is purely a hook to 367 // allow some executors (such as the combiner) to check arbitrary, 368 // contextless predicates, such as whether a rule is enabled or not. 369 int64_t Predicate = MatchTable[CurrentIdx++]; 370 DEBUG_WITH_TYPE(TgtExecutor::getName(), 371 dbgs() << CurrentIdx 372 << ": GIM_CheckSimplePredicate(Predicate=" 373 << Predicate << ")\n"); 374 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 375 if (!testSimplePredicate(Predicate)) { 376 if (handleReject() == RejectAndGiveUp) 377 return false; 378 } 379 break; 380 } 381 case GIM_CheckCxxInsnPredicate: { 382 int64_t InsnID = MatchTable[CurrentIdx++]; 383 int64_t Predicate = MatchTable[CurrentIdx++]; 384 DEBUG_WITH_TYPE(TgtExecutor::getName(), 385 dbgs() 386 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" 387 << InsnID << "], Predicate=" << Predicate << ")\n"); 388 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 389 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 390 391 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) 392 if (handleReject() == RejectAndGiveUp) 393 return false; 394 break; 395 } 396 case GIM_CheckHasNoUse: { 397 int64_t InsnID = MatchTable[CurrentIdx++]; 398 399 DEBUG_WITH_TYPE(TgtExecutor::getName(), 400 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" 401 << InsnID << "]\n"); 402 403 const MachineInstr *MI = State.MIs[InsnID]; 404 assert(MI && "Used insn before defined"); 405 assert(MI->getNumDefs() > 0 && "No defs"); 406 const Register Res = MI->getOperand(0).getReg(); 407 408 if (!MRI.use_nodbg_empty(Res)) { 409 if (handleReject() == RejectAndGiveUp) 410 return false; 411 } 412 413 break; 414 } 415 case GIM_CheckAtomicOrdering: { 416 int64_t InsnID = MatchTable[CurrentIdx++]; 417 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 418 DEBUG_WITH_TYPE(TgtExecutor::getName(), 419 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" 420 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 421 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 422 if (!State.MIs[InsnID]->hasOneMemOperand()) 423 if (handleReject() == RejectAndGiveUp) 424 return false; 425 426 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 427 if (MMO->getMergedOrdering() != Ordering) 428 if (handleReject() == RejectAndGiveUp) 429 return false; 430 break; 431 } 432 case GIM_CheckAtomicOrderingOrStrongerThan: { 433 int64_t InsnID = MatchTable[CurrentIdx++]; 434 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 435 DEBUG_WITH_TYPE(TgtExecutor::getName(), 436 dbgs() << CurrentIdx 437 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" 438 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 439 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 440 if (!State.MIs[InsnID]->hasOneMemOperand()) 441 if (handleReject() == RejectAndGiveUp) 442 return false; 443 444 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 445 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) 446 if (handleReject() == RejectAndGiveUp) 447 return false; 448 break; 449 } 450 case GIM_CheckAtomicOrderingWeakerThan: { 451 int64_t InsnID = MatchTable[CurrentIdx++]; 452 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 453 DEBUG_WITH_TYPE(TgtExecutor::getName(), 454 dbgs() << CurrentIdx 455 << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" 456 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 457 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 458 if (!State.MIs[InsnID]->hasOneMemOperand()) 459 if (handleReject() == RejectAndGiveUp) 460 return false; 461 462 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 463 if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) 464 if (handleReject() == RejectAndGiveUp) 465 return false; 466 break; 467 } 468 case GIM_CheckMemoryAddressSpace: { 469 int64_t InsnID = MatchTable[CurrentIdx++]; 470 int64_t MMOIdx = MatchTable[CurrentIdx++]; 471 // This accepts a list of possible address spaces. 472 const int NumAddrSpace = MatchTable[CurrentIdx++]; 473 474 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 475 if (handleReject() == RejectAndGiveUp) 476 return false; 477 break; 478 } 479 480 // Need to still jump to the end of the list of address spaces if we find 481 // a match earlier. 482 const uint64_t LastIdx = CurrentIdx + NumAddrSpace; 483 484 const MachineMemOperand *MMO = 485 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 486 const unsigned MMOAddrSpace = MMO->getAddrSpace(); 487 488 bool Success = false; 489 for (int I = 0; I != NumAddrSpace; ++I) { 490 unsigned AddrSpace = MatchTable[CurrentIdx++]; 491 DEBUG_WITH_TYPE(TgtExecutor::getName(), 492 dbgs() << "addrspace(" << MMOAddrSpace << ") vs " 493 << AddrSpace << '\n'); 494 495 if (AddrSpace == MMOAddrSpace) { 496 Success = true; 497 break; 498 } 499 } 500 501 CurrentIdx = LastIdx; 502 if (!Success && handleReject() == RejectAndGiveUp) 503 return false; 504 break; 505 } 506 case GIM_CheckMemoryAlignment: { 507 int64_t InsnID = MatchTable[CurrentIdx++]; 508 int64_t MMOIdx = MatchTable[CurrentIdx++]; 509 unsigned MinAlign = MatchTable[CurrentIdx++]; 510 511 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 512 513 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 514 if (handleReject() == RejectAndGiveUp) 515 return false; 516 break; 517 } 518 519 MachineMemOperand *MMO = 520 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 521 DEBUG_WITH_TYPE(TgtExecutor::getName(), 522 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" 523 << "(MIs[" << InsnID << "]->memoperands() + " 524 << MMOIdx << ")->getAlignment() >= " << MinAlign 525 << ")\n"); 526 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) 527 return false; 528 529 break; 530 } 531 case GIM_CheckMemorySizeEqualTo: { 532 int64_t InsnID = MatchTable[CurrentIdx++]; 533 int64_t MMOIdx = MatchTable[CurrentIdx++]; 534 uint64_t Size = MatchTable[CurrentIdx++]; 535 536 DEBUG_WITH_TYPE(TgtExecutor::getName(), 537 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" 538 << InsnID << "]->memoperands() + " << MMOIdx 539 << ", Size=" << Size << ")\n"); 540 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 541 542 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 543 if (handleReject() == RejectAndGiveUp) 544 return false; 545 break; 546 } 547 548 MachineMemOperand *MMO = 549 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 550 551 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() 552 << " bytes vs " << Size 553 << " bytes\n"); 554 if (MMO->getSize() != Size) 555 if (handleReject() == RejectAndGiveUp) 556 return false; 557 558 break; 559 } 560 case GIM_CheckMemorySizeEqualToLLT: 561 case GIM_CheckMemorySizeLessThanLLT: 562 case GIM_CheckMemorySizeGreaterThanLLT: { 563 int64_t InsnID = MatchTable[CurrentIdx++]; 564 int64_t MMOIdx = MatchTable[CurrentIdx++]; 565 int64_t OpIdx = MatchTable[CurrentIdx++]; 566 567 DEBUG_WITH_TYPE( 568 TgtExecutor::getName(), 569 dbgs() << CurrentIdx << ": GIM_CheckMemorySize" 570 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" 571 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT 572 ? "GreaterThan" 573 : "LessThan") 574 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 575 << ", OpIdx=" << OpIdx << ")\n"); 576 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 577 578 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 579 if (!MO.isReg()) { 580 DEBUG_WITH_TYPE(TgtExecutor::getName(), 581 dbgs() << CurrentIdx << ": Not a register\n"); 582 if (handleReject() == RejectAndGiveUp) 583 return false; 584 break; 585 } 586 587 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 588 if (handleReject() == RejectAndGiveUp) 589 return false; 590 break; 591 } 592 593 MachineMemOperand *MMO = 594 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 595 596 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); 597 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && 598 MMO->getSizeInBits() != Size) { 599 if (handleReject() == RejectAndGiveUp) 600 return false; 601 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && 602 MMO->getSizeInBits() >= Size) { 603 if (handleReject() == RejectAndGiveUp) 604 return false; 605 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && 606 MMO->getSizeInBits() <= Size) 607 if (handleReject() == RejectAndGiveUp) 608 return false; 609 610 break; 611 } 612 case GIM_CheckType: { 613 int64_t InsnID = MatchTable[CurrentIdx++]; 614 int64_t OpIdx = MatchTable[CurrentIdx++]; 615 int64_t TypeID = MatchTable[CurrentIdx++]; 616 DEBUG_WITH_TYPE(TgtExecutor::getName(), 617 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID 618 << "]->getOperand(" << OpIdx 619 << "), TypeID=" << TypeID << ")\n"); 620 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 621 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 622 if (!MO.isReg() || 623 MRI.getType(MO.getReg()) != ExecInfo.TypeObjects[TypeID]) { 624 if (handleReject() == RejectAndGiveUp) 625 return false; 626 } 627 break; 628 } 629 case GIM_CheckPointerToAny: { 630 int64_t InsnID = MatchTable[CurrentIdx++]; 631 int64_t OpIdx = MatchTable[CurrentIdx++]; 632 uint64_t SizeInBits = MatchTable[CurrentIdx++]; 633 634 DEBUG_WITH_TYPE(TgtExecutor::getName(), 635 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" 636 << InsnID << "]->getOperand(" << OpIdx 637 << "), SizeInBits=" << SizeInBits << ")\n"); 638 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 639 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 640 const LLT Ty = MRI.getType(MO.getReg()); 641 642 // iPTR must be looked up in the target. 643 if (SizeInBits == 0) { 644 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); 645 const unsigned AddrSpace = Ty.getAddressSpace(); 646 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); 647 } 648 649 assert(SizeInBits != 0 && "Pointer size must be known"); 650 651 if (MO.isReg()) { 652 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) 653 if (handleReject() == RejectAndGiveUp) 654 return false; 655 } else if (handleReject() == RejectAndGiveUp) 656 return false; 657 658 break; 659 } 660 case GIM_RecordNamedOperand: { 661 int64_t InsnID = MatchTable[CurrentIdx++]; 662 int64_t OpIdx = MatchTable[CurrentIdx++]; 663 uint64_t StoreIdx = MatchTable[CurrentIdx++]; 664 665 DEBUG_WITH_TYPE(TgtExecutor::getName(), 666 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" 667 << InsnID << "]->getOperand(" << OpIdx 668 << "), StoreIdx=" << StoreIdx << ")\n"); 669 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 670 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); 671 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); 672 break; 673 } 674 case GIM_CheckRegBankForClass: { 675 int64_t InsnID = MatchTable[CurrentIdx++]; 676 int64_t OpIdx = MatchTable[CurrentIdx++]; 677 int64_t RCEnum = MatchTable[CurrentIdx++]; 678 DEBUG_WITH_TYPE(TgtExecutor::getName(), 679 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" 680 << InsnID << "]->getOperand(" << OpIdx 681 << "), RCEnum=" << RCEnum << ")\n"); 682 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 683 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 684 if (!MO.isReg() || 685 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), 686 MRI.getType(MO.getReg())) != 687 RBI.getRegBank(MO.getReg(), MRI, TRI)) { 688 if (handleReject() == RejectAndGiveUp) 689 return false; 690 } 691 break; 692 } 693 694 case GIM_CheckComplexPattern: { 695 int64_t InsnID = MatchTable[CurrentIdx++]; 696 int64_t OpIdx = MatchTable[CurrentIdx++]; 697 int64_t RendererID = MatchTable[CurrentIdx++]; 698 int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; 699 DEBUG_WITH_TYPE(TgtExecutor::getName(), 700 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID 701 << "] = GIM_CheckComplexPattern(MIs[" << InsnID 702 << "]->getOperand(" << OpIdx 703 << "), ComplexPredicateID=" << ComplexPredicateID 704 << ")\n"); 705 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 706 // FIXME: Use std::invoke() when it's available. 707 ComplexRendererFns Renderer = 708 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( 709 State.MIs[InsnID]->getOperand(OpIdx)); 710 if (Renderer) 711 State.Renderers[RendererID] = *Renderer; 712 else if (handleReject() == RejectAndGiveUp) 713 return false; 714 break; 715 } 716 717 case GIM_CheckConstantInt: { 718 int64_t InsnID = MatchTable[CurrentIdx++]; 719 int64_t OpIdx = MatchTable[CurrentIdx++]; 720 int64_t Value = MatchTable[CurrentIdx++]; 721 DEBUG_WITH_TYPE(TgtExecutor::getName(), 722 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" 723 << InsnID << "]->getOperand(" << OpIdx 724 << "), Value=" << Value << ")\n"); 725 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 726 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 727 if (MO.isReg()) { 728 // isOperandImmEqual() will sign-extend to 64-bits, so should we. 729 LLT Ty = MRI.getType(MO.getReg()); 730 Value = SignExtend64(Value, Ty.getSizeInBits()); 731 732 if (!isOperandImmEqual(MO, Value, MRI)) { 733 if (handleReject() == RejectAndGiveUp) 734 return false; 735 } 736 } else if (handleReject() == RejectAndGiveUp) 737 return false; 738 739 break; 740 } 741 742 case GIM_CheckLiteralInt: { 743 int64_t InsnID = MatchTable[CurrentIdx++]; 744 int64_t OpIdx = MatchTable[CurrentIdx++]; 745 int64_t Value = MatchTable[CurrentIdx++]; 746 DEBUG_WITH_TYPE(TgtExecutor::getName(), 747 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" 748 << InsnID << "]->getOperand(" << OpIdx 749 << "), Value=" << Value << ")\n"); 750 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 751 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 752 if (MO.isImm() && MO.getImm() == Value) 753 break; 754 755 if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) 756 break; 757 758 if (handleReject() == RejectAndGiveUp) 759 return false; 760 761 break; 762 } 763 764 case GIM_CheckIntrinsicID: { 765 int64_t InsnID = MatchTable[CurrentIdx++]; 766 int64_t OpIdx = MatchTable[CurrentIdx++]; 767 int64_t Value = MatchTable[CurrentIdx++]; 768 DEBUG_WITH_TYPE(TgtExecutor::getName(), 769 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" 770 << InsnID << "]->getOperand(" << OpIdx 771 << "), Value=" << Value << ")\n"); 772 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 773 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 774 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) 775 if (handleReject() == RejectAndGiveUp) 776 return false; 777 break; 778 } 779 case GIM_CheckCmpPredicate: { 780 int64_t InsnID = MatchTable[CurrentIdx++]; 781 int64_t OpIdx = MatchTable[CurrentIdx++]; 782 int64_t Value = MatchTable[CurrentIdx++]; 783 DEBUG_WITH_TYPE(TgtExecutor::getName(), 784 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" 785 << InsnID << "]->getOperand(" << OpIdx 786 << "), Value=" << Value << ")\n"); 787 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 788 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 789 if (!MO.isPredicate() || MO.getPredicate() != Value) 790 if (handleReject() == RejectAndGiveUp) 791 return false; 792 break; 793 } 794 case GIM_CheckIsMBB: { 795 int64_t InsnID = MatchTable[CurrentIdx++]; 796 int64_t OpIdx = MatchTable[CurrentIdx++]; 797 DEBUG_WITH_TYPE(TgtExecutor::getName(), 798 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID 799 << "]->getOperand(" << OpIdx << "))\n"); 800 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 801 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { 802 if (handleReject() == RejectAndGiveUp) 803 return false; 804 } 805 break; 806 } 807 case GIM_CheckIsImm: { 808 int64_t InsnID = MatchTable[CurrentIdx++]; 809 int64_t OpIdx = MatchTable[CurrentIdx++]; 810 DEBUG_WITH_TYPE(TgtExecutor::getName(), 811 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID 812 << "]->getOperand(" << OpIdx << "))\n"); 813 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 814 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { 815 if (handleReject() == RejectAndGiveUp) 816 return false; 817 } 818 break; 819 } 820 case GIM_CheckIsSafeToFold: { 821 int64_t InsnID = MatchTable[CurrentIdx++]; 822 DEBUG_WITH_TYPE(TgtExecutor::getName(), 823 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" 824 << InsnID << "])\n"); 825 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 826 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { 827 if (handleReject() == RejectAndGiveUp) 828 return false; 829 } 830 break; 831 } 832 case GIM_CheckIsSameOperand: 833 case GIM_CheckIsSameOperandIgnoreCopies: { 834 int64_t InsnID = MatchTable[CurrentIdx++]; 835 int64_t OpIdx = MatchTable[CurrentIdx++]; 836 int64_t OtherInsnID = MatchTable[CurrentIdx++]; 837 int64_t OtherOpIdx = MatchTable[CurrentIdx++]; 838 DEBUG_WITH_TYPE(TgtExecutor::getName(), 839 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" 840 << InsnID << "][" << OpIdx << "], MIs[" 841 << OtherInsnID << "][" << OtherOpIdx << "])\n"); 842 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 843 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); 844 845 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); 846 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); 847 848 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { 849 if (Op.isReg() && OtherOp.isReg()) { 850 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) == 851 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI)) 852 break; 853 } 854 } 855 856 if (!Op.isIdenticalTo(OtherOp)) { 857 if (handleReject() == RejectAndGiveUp) 858 return false; 859 } 860 break; 861 } 862 case GIM_Reject: 863 DEBUG_WITH_TYPE(TgtExecutor::getName(), 864 dbgs() << CurrentIdx << ": GIM_Reject\n"); 865 if (handleReject() == RejectAndGiveUp) 866 return false; 867 break; 868 case GIR_MutateOpcode: { 869 int64_t OldInsnID = MatchTable[CurrentIdx++]; 870 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 871 int64_t NewOpcode = MatchTable[CurrentIdx++]; 872 if (NewInsnID >= OutMIs.size()) 873 OutMIs.resize(NewInsnID + 1); 874 875 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), 876 State.MIs[OldInsnID]); 877 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); 878 DEBUG_WITH_TYPE(TgtExecutor::getName(), 879 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" 880 << NewInsnID << "], MIs[" << OldInsnID << "], " 881 << NewOpcode << ")\n"); 882 break; 883 } 884 885 case GIR_BuildMI: { 886 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 887 int64_t Opcode = MatchTable[CurrentIdx++]; 888 if (NewInsnID >= OutMIs.size()) 889 OutMIs.resize(NewInsnID + 1); 890 891 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], 892 MIMetadata(*State.MIs[0]), TII.get(Opcode)); 893 DEBUG_WITH_TYPE(TgtExecutor::getName(), 894 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" 895 << NewInsnID << "], " << Opcode << ")\n"); 896 break; 897 } 898 899 case GIR_Copy: { 900 int64_t NewInsnID = MatchTable[CurrentIdx++]; 901 int64_t OldInsnID = MatchTable[CurrentIdx++]; 902 int64_t OpIdx = MatchTable[CurrentIdx++]; 903 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 904 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); 905 DEBUG_WITH_TYPE(TgtExecutor::getName(), 906 dbgs() 907 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID 908 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); 909 break; 910 } 911 912 case GIR_CopyOrAddZeroReg: { 913 int64_t NewInsnID = MatchTable[CurrentIdx++]; 914 int64_t OldInsnID = MatchTable[CurrentIdx++]; 915 int64_t OpIdx = MatchTable[CurrentIdx++]; 916 int64_t ZeroReg = MatchTable[CurrentIdx++]; 917 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 918 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); 919 if (isOperandImmEqual(MO, 0, MRI)) 920 OutMIs[NewInsnID].addReg(ZeroReg); 921 else 922 OutMIs[NewInsnID].add(MO); 923 DEBUG_WITH_TYPE(TgtExecutor::getName(), 924 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" 925 << NewInsnID << "], MIs[" << OldInsnID << "], " 926 << OpIdx << ", " << ZeroReg << ")\n"); 927 break; 928 } 929 930 case GIR_CopySubReg: { 931 int64_t NewInsnID = MatchTable[CurrentIdx++]; 932 int64_t OldInsnID = MatchTable[CurrentIdx++]; 933 int64_t OpIdx = MatchTable[CurrentIdx++]; 934 int64_t SubRegIdx = MatchTable[CurrentIdx++]; 935 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 936 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 937 0, SubRegIdx); 938 DEBUG_WITH_TYPE(TgtExecutor::getName(), 939 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" 940 << NewInsnID << "], MIs[" << OldInsnID << "], " 941 << OpIdx << ", " << SubRegIdx << ")\n"); 942 break; 943 } 944 945 case GIR_AddImplicitDef: { 946 int64_t InsnID = MatchTable[CurrentIdx++]; 947 int64_t RegNum = MatchTable[CurrentIdx++]; 948 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 949 OutMIs[InsnID].addDef(RegNum, RegState::Implicit); 950 DEBUG_WITH_TYPE(TgtExecutor::getName(), 951 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" 952 << InsnID << "], " << RegNum << ")\n"); 953 break; 954 } 955 956 case GIR_AddImplicitUse: { 957 int64_t InsnID = MatchTable[CurrentIdx++]; 958 int64_t RegNum = MatchTable[CurrentIdx++]; 959 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 960 OutMIs[InsnID].addUse(RegNum, RegState::Implicit); 961 DEBUG_WITH_TYPE(TgtExecutor::getName(), 962 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" 963 << InsnID << "], " << RegNum << ")\n"); 964 break; 965 } 966 967 case GIR_AddRegister: { 968 int64_t InsnID = MatchTable[CurrentIdx++]; 969 int64_t RegNum = MatchTable[CurrentIdx++]; 970 uint64_t RegFlags = MatchTable[CurrentIdx++]; 971 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 972 OutMIs[InsnID].addReg(RegNum, RegFlags); 973 DEBUG_WITH_TYPE(TgtExecutor::getName(), 974 dbgs() 975 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID 976 << "], " << RegNum << ", " << RegFlags << ")\n"); 977 break; 978 } 979 980 case GIR_AddTempRegister: 981 case GIR_AddTempSubRegister: { 982 int64_t InsnID = MatchTable[CurrentIdx++]; 983 int64_t TempRegID = MatchTable[CurrentIdx++]; 984 uint64_t TempRegFlags = MatchTable[CurrentIdx++]; 985 unsigned SubReg = 0; 986 if (MatcherOpcode == GIR_AddTempSubRegister) 987 SubReg = MatchTable[CurrentIdx++]; 988 989 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 990 991 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, 992 SubReg); 993 DEBUG_WITH_TYPE( 994 TgtExecutor::getName(), 995 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID 996 << "], TempRegisters[" << TempRegID << "]"; 997 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); 998 dbgs() << ", " << TempRegFlags << ")\n"); 999 break; 1000 } 1001 1002 case GIR_AddImm: { 1003 int64_t InsnID = MatchTable[CurrentIdx++]; 1004 int64_t Imm = MatchTable[CurrentIdx++]; 1005 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1006 OutMIs[InsnID].addImm(Imm); 1007 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1008 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID 1009 << "], " << Imm << ")\n"); 1010 break; 1011 } 1012 1013 case GIR_ComplexRenderer: { 1014 int64_t InsnID = MatchTable[CurrentIdx++]; 1015 int64_t RendererID = MatchTable[CurrentIdx++]; 1016 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1017 for (const auto &RenderOpFn : State.Renderers[RendererID]) 1018 RenderOpFn(OutMIs[InsnID]); 1019 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1020 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" 1021 << InsnID << "], " << RendererID << ")\n"); 1022 break; 1023 } 1024 case GIR_ComplexSubOperandRenderer: { 1025 int64_t InsnID = MatchTable[CurrentIdx++]; 1026 int64_t RendererID = MatchTable[CurrentIdx++]; 1027 int64_t RenderOpID = MatchTable[CurrentIdx++]; 1028 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1029 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); 1030 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1031 dbgs() << CurrentIdx 1032 << ": GIR_ComplexSubOperandRenderer(OutMIs[" 1033 << InsnID << "], " << RendererID << ", " 1034 << RenderOpID << ")\n"); 1035 break; 1036 } 1037 case GIR_ComplexSubOperandSubRegRenderer: { 1038 int64_t InsnID = MatchTable[CurrentIdx++]; 1039 int64_t RendererID = MatchTable[CurrentIdx++]; 1040 int64_t RenderOpID = MatchTable[CurrentIdx++]; 1041 int64_t SubRegIdx = MatchTable[CurrentIdx++]; 1042 MachineInstrBuilder &MI = OutMIs[InsnID]; 1043 assert(MI && "Attempted to add to undefined instruction"); 1044 State.Renderers[RendererID][RenderOpID](MI); 1045 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx); 1046 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1047 dbgs() << CurrentIdx 1048 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" 1049 << InsnID << "], " << RendererID << ", " 1050 << RenderOpID << ", " << SubRegIdx << ")\n"); 1051 break; 1052 } 1053 1054 case GIR_CopyConstantAsSImm: { 1055 int64_t NewInsnID = MatchTable[CurrentIdx++]; 1056 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1057 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1058 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 1059 "Expected G_CONSTANT"); 1060 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { 1061 OutMIs[NewInsnID].addImm( 1062 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); 1063 } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) 1064 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); 1065 else 1066 llvm_unreachable("Expected Imm or CImm operand"); 1067 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1068 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" 1069 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1070 break; 1071 } 1072 1073 // TODO: Needs a test case once we have a pattern that uses this. 1074 case GIR_CopyFConstantAsFPImm: { 1075 int64_t NewInsnID = MatchTable[CurrentIdx++]; 1076 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1077 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1078 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 1079 "Expected G_FCONSTANT"); 1080 if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) 1081 OutMIs[NewInsnID].addFPImm( 1082 State.MIs[OldInsnID]->getOperand(1).getFPImm()); 1083 else 1084 llvm_unreachable("Expected FPImm operand"); 1085 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1086 dbgs() 1087 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" 1088 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1089 break; 1090 } 1091 1092 case GIR_CustomRenderer: { 1093 int64_t InsnID = MatchTable[CurrentIdx++]; 1094 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1095 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1096 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1097 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1098 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" 1099 << InsnID << "], MIs[" << OldInsnID << "], " 1100 << RendererFnID << ")\n"); 1101 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1102 OutMIs[InsnID], *State.MIs[OldInsnID], 1103 -1); // Not a source operand of the old instruction. 1104 break; 1105 } 1106 case GIR_CustomAction: { 1107 int64_t FnID = MatchTable[CurrentIdx++]; 1108 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1109 dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID 1110 << ")\n"); 1111 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID"); 1112 runCustomAction(FnID, State); 1113 break; 1114 } 1115 case GIR_CustomOperandRenderer: { 1116 int64_t InsnID = MatchTable[CurrentIdx++]; 1117 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1118 int64_t OpIdx = MatchTable[CurrentIdx++]; 1119 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1120 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1121 1122 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1123 dbgs() << CurrentIdx 1124 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID 1125 << "], MIs[" << OldInsnID << "]->getOperand(" 1126 << OpIdx << "), " << RendererFnID << ")\n"); 1127 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1128 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); 1129 break; 1130 } 1131 case GIR_ConstrainOperandRC: { 1132 int64_t InsnID = MatchTable[CurrentIdx++]; 1133 int64_t OpIdx = MatchTable[CurrentIdx++]; 1134 int64_t RCEnum = MatchTable[CurrentIdx++]; 1135 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1136 MachineInstr &I = *OutMIs[InsnID].getInstr(); 1137 MachineFunction &MF = *I.getParent()->getParent(); 1138 MachineRegisterInfo &MRI = MF.getRegInfo(); 1139 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); 1140 MachineOperand &MO = I.getOperand(OpIdx); 1141 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); 1142 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1143 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" 1144 << InsnID << "], " << OpIdx << ", " << RCEnum 1145 << ")\n"); 1146 break; 1147 } 1148 1149 case GIR_ConstrainSelectedInstOperands: { 1150 int64_t InsnID = MatchTable[CurrentIdx++]; 1151 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1152 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, 1153 RBI); 1154 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1155 dbgs() << CurrentIdx 1156 << ": GIR_ConstrainSelectedInstOperands(OutMIs[" 1157 << InsnID << "])\n"); 1158 break; 1159 } 1160 1161 case GIR_MergeMemOperands: { 1162 int64_t InsnID = MatchTable[CurrentIdx++]; 1163 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1164 1165 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1166 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" 1167 << InsnID << "]"); 1168 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; 1169 while ((MergeInsnID = MatchTable[CurrentIdx++]) != 1170 GIU_MergeMemOperands_EndOfList) { 1171 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1172 dbgs() << ", MIs[" << MergeInsnID << "]"); 1173 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) 1174 OutMIs[InsnID].addMemOperand(MMO); 1175 } 1176 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n"); 1177 break; 1178 } 1179 1180 case GIR_EraseFromParent: { 1181 int64_t InsnID = MatchTable[CurrentIdx++]; 1182 assert(State.MIs[InsnID] && 1183 "Attempted to erase an undefined instruction"); 1184 State.MIs[InsnID]->eraseFromParent(); 1185 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1186 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" 1187 << InsnID << "])\n"); 1188 break; 1189 } 1190 1191 case GIR_MakeTempReg: { 1192 int64_t TempRegID = MatchTable[CurrentIdx++]; 1193 int64_t TypeID = MatchTable[CurrentIdx++]; 1194 1195 State.TempRegisters[TempRegID] = 1196 MRI.createGenericVirtualRegister(ExecInfo.TypeObjects[TypeID]); 1197 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1198 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID 1199 << "] = GIR_MakeTempReg(" << TypeID << ")\n"); 1200 break; 1201 } 1202 1203 case GIR_Coverage: { 1204 int64_t RuleID = MatchTable[CurrentIdx++]; 1205 assert(CoverageInfo); 1206 CoverageInfo->setCovered(RuleID); 1207 1208 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx 1209 << ": GIR_Coverage(" 1210 << RuleID << ")"); 1211 break; 1212 } 1213 1214 case GIR_Done: 1215 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1216 dbgs() << CurrentIdx << ": GIR_Done\n"); 1217 propagateFlags(OutMIs); 1218 return true; 1219 default: 1220 llvm_unreachable("Unexpected command"); 1221 } 1222 } 1223 } 1224 1225 } // end namespace llvm 1226 1227 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 1228