1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable` 10 /// function. This is implemented in a separate file because the function is 11 /// quite large. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 17 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" 20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 22 #include "llvm/CodeGen/GlobalISel/Utils.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineOperand.h" 25 #include "llvm/CodeGen/MachineRegisterInfo.h" 26 #include "llvm/CodeGen/RegisterBankInfo.h" 27 #include "llvm/CodeGen/TargetInstrInfo.h" 28 #include "llvm/CodeGen/TargetOpcodes.h" 29 #include "llvm/CodeGen/TargetRegisterInfo.h" 30 #include "llvm/IR/Constants.h" 31 #include "llvm/IR/DataLayout.h" 32 #include "llvm/IR/Type.h" 33 #include "llvm/Support/CodeGenCoverage.h" 34 #include "llvm/Support/Debug.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/LEB128.h" 37 #include "llvm/Support/raw_ostream.h" 38 #include <cassert> 39 #include <cstddef> 40 #include <cstdint> 41 42 namespace llvm { 43 44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, 45 class CustomRendererFn> 46 bool GIMatchTableExecutor::executeMatchTable( 47 TgtExecutor &Exec, MatcherState &State, 48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> 49 &ExecInfo, 50 MachineIRBuilder &Builder, const uint8_t *MatchTable, 51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI, 52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, 53 const PredicateBitset &AvailableFeatures, 54 CodeGenCoverage *CoverageInfo) const { 55 56 uint64_t CurrentIdx = 0; 57 SmallVector<uint64_t, 4> OnFailResumeAt; 58 NewMIVector OutMIs; 59 60 GISelChangeObserver *Observer = Builder.getObserver(); 61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc. 62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); 63 64 const uint16_t Flags = State.MIs[0]->getFlags(); 65 66 enum RejectAction { RejectAndGiveUp, RejectAndResume }; 67 auto handleReject = [&]() -> RejectAction { 68 DEBUG_WITH_TYPE(TgtExecutor::getName(), 69 dbgs() << CurrentIdx << ": Rejected\n"); 70 if (OnFailResumeAt.empty()) 71 return RejectAndGiveUp; 72 CurrentIdx = OnFailResumeAt.pop_back_val(); 73 DEBUG_WITH_TYPE(TgtExecutor::getName(), 74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" 75 << OnFailResumeAt.size() << " try-blocks remain)\n"); 76 return RejectAndResume; 77 }; 78 79 const auto propagateFlags = [&]() { 80 for (auto MIB : OutMIs) { 81 // Set the NoFPExcept flag when no original matched instruction could 82 // raise an FP exception, but the new instruction potentially might. 83 uint16_t MIBFlags = Flags; 84 if (NoFPException && MIB->mayRaiseFPException()) 85 MIBFlags |= MachineInstr::NoFPExcept; 86 if (Observer) 87 Observer->changingInstr(*MIB); 88 MIB.setMIFlags(MIBFlags); 89 if (Observer) 90 Observer->changedInstr(*MIB); 91 } 92 }; 93 94 // If the index is >= 0, it's an index in the type objects generated by 95 // TableGen. If the index is <0, it's an index in the recorded types object. 96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT { 97 if (Idx >= 0) 98 return ExecInfo.TypeObjects[Idx]; 99 return State.RecordedTypes[1 - Idx]; 100 }; 101 102 const auto readULEB = [&]() { 103 unsigned N = 0; 104 uint64_t Val = decodeULEB128(MatchTable + CurrentIdx, &N); 105 CurrentIdx += N; 106 return Val; 107 }; 108 109 // Convenience function to return a signed value. This avoids 110 // us forgetting to first cast to int8_t before casting to a 111 // wider signed int type. 112 // if we casted uint8 directly to a wider type we'd lose 113 // negative values. 114 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; }; 115 116 const auto readU16 = [&]() { 117 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx); 118 CurrentIdx += 2; 119 return V; 120 }; 121 122 const auto readU32 = [&]() { 123 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx); 124 CurrentIdx += 4; 125 return V; 126 }; 127 128 const auto readU64 = [&]() { 129 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx); 130 CurrentIdx += 8; 131 return V; 132 }; 133 134 while (true) { 135 assert(CurrentIdx != ~0u && "Invalid MatchTable index"); 136 uint8_t MatcherOpcode = MatchTable[CurrentIdx++]; 137 switch (MatcherOpcode) { 138 case GIM_Try: { 139 DEBUG_WITH_TYPE(TgtExecutor::getName(), 140 dbgs() << CurrentIdx << ": Begin try-block\n"); 141 OnFailResumeAt.push_back(readU32()); 142 break; 143 } 144 145 case GIM_RecordInsn: 146 case GIM_RecordInsnIgnoreCopies: { 147 uint64_t NewInsnID = readULEB(); 148 uint64_t InsnID = readULEB(); 149 uint64_t OpIdx = readULEB(); 150 151 // As an optimisation we require that MIs[0] is always the root. Refuse 152 // any attempt to modify it. 153 assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); 154 155 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 156 if (!MO.isReg()) { 157 DEBUG_WITH_TYPE(TgtExecutor::getName(), 158 dbgs() << CurrentIdx << ": Not a register\n"); 159 if (handleReject() == RejectAndGiveUp) 160 return false; 161 break; 162 } 163 if (MO.getReg().isPhysical()) { 164 DEBUG_WITH_TYPE(TgtExecutor::getName(), 165 dbgs() << CurrentIdx << ": Is a physical register\n"); 166 if (handleReject() == RejectAndGiveUp) 167 return false; 168 break; 169 } 170 171 MachineInstr *NewMI; 172 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) 173 NewMI = getDefIgnoringCopies(MO.getReg(), MRI); 174 else 175 NewMI = MRI.getVRegDef(MO.getReg()); 176 177 if ((size_t)NewInsnID < State.MIs.size()) 178 State.MIs[NewInsnID] = NewMI; 179 else { 180 assert((size_t)NewInsnID == State.MIs.size() && 181 "Expected to store MIs in order"); 182 State.MIs.push_back(NewMI); 183 } 184 DEBUG_WITH_TYPE(TgtExecutor::getName(), 185 dbgs() << CurrentIdx << ": MIs[" << NewInsnID 186 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx 187 << ")\n"); 188 break; 189 } 190 191 case GIM_CheckFeatures: { 192 uint16_t ExpectedBitsetID = readU16(); 193 DEBUG_WITH_TYPE(TgtExecutor::getName(), 194 dbgs() << CurrentIdx 195 << ": GIM_CheckFeatures(ExpectedBitsetID=" 196 << ExpectedBitsetID << ")\n"); 197 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != 198 ExecInfo.FeatureBitsets[ExpectedBitsetID]) { 199 if (handleReject() == RejectAndGiveUp) 200 return false; 201 } 202 break; 203 } 204 case GIM_CheckOpcode: 205 case GIM_CheckOpcodeIsEither: { 206 uint64_t InsnID = readULEB(); 207 uint16_t Expected0 = readU16(); 208 uint16_t Expected1 = -1; 209 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 210 Expected1 = readU16(); 211 212 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 213 unsigned Opcode = State.MIs[InsnID]->getOpcode(); 214 215 DEBUG_WITH_TYPE(TgtExecutor::getName(), 216 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID 217 << "], ExpectedOpcode=" << Expected0; 218 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() 219 << " || " << Expected1; 220 dbgs() << ") // Got=" << Opcode << "\n";); 221 222 if (Opcode != Expected0 && Opcode != Expected1) { 223 if (handleReject() == RejectAndGiveUp) 224 return false; 225 } 226 break; 227 } 228 case GIM_SwitchOpcode: { 229 uint64_t InsnID = readULEB(); 230 uint16_t LowerBound = readU16(); 231 uint16_t UpperBound = readU16(); 232 uint32_t Default = readU32(); 233 234 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 235 const int64_t Opcode = State.MIs[InsnID]->getOpcode(); 236 237 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 238 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" 239 << LowerBound << ", " << UpperBound << "), Default=" << Default 240 << ", JumpTable...) // Got=" << Opcode << "\n"; 241 }); 242 if (Opcode < LowerBound || UpperBound <= Opcode) { 243 CurrentIdx = Default; 244 break; 245 } 246 const auto EntryIdx = (Opcode - LowerBound); 247 // Each entry is 4 bytes 248 CurrentIdx = 249 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4)); 250 if (!CurrentIdx) { 251 CurrentIdx = Default; 252 break; 253 } 254 OnFailResumeAt.push_back(Default); 255 break; 256 } 257 258 case GIM_SwitchType: { 259 uint64_t InsnID = readULEB(); 260 uint64_t OpIdx = readULEB(); 261 uint16_t LowerBound = readU16(); 262 uint16_t UpperBound = readU16(); 263 int64_t Default = readU32(); 264 265 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 266 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 267 268 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 269 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID 270 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " 271 << UpperBound << "), Default=" << Default 272 << ", JumpTable...) // Got="; 273 if (!MO.isReg()) 274 dbgs() << "Not a VReg\n"; 275 else 276 dbgs() << MRI.getType(MO.getReg()) << "\n"; 277 }); 278 if (!MO.isReg()) { 279 CurrentIdx = Default; 280 break; 281 } 282 const LLT Ty = MRI.getType(MO.getReg()); 283 const auto TyI = ExecInfo.TypeIDMap.find(Ty); 284 if (TyI == ExecInfo.TypeIDMap.end()) { 285 CurrentIdx = Default; 286 break; 287 } 288 const int64_t TypeID = TyI->second; 289 if (TypeID < LowerBound || UpperBound <= TypeID) { 290 CurrentIdx = Default; 291 break; 292 } 293 const auto NumEntry = (TypeID - LowerBound); 294 // Each entry is 4 bytes 295 CurrentIdx = 296 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4)); 297 if (!CurrentIdx) { 298 CurrentIdx = Default; 299 break; 300 } 301 OnFailResumeAt.push_back(Default); 302 break; 303 } 304 305 case GIM_CheckNumOperands: { 306 uint64_t InsnID = readULEB(); 307 uint64_t Expected = readULEB(); 308 DEBUG_WITH_TYPE(TgtExecutor::getName(), 309 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" 310 << InsnID << "], Expected=" << Expected << ")\n"); 311 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 312 if (State.MIs[InsnID]->getNumOperands() != Expected) { 313 if (handleReject() == RejectAndGiveUp) 314 return false; 315 } 316 break; 317 } 318 case GIM_CheckI64ImmPredicate: 319 case GIM_CheckImmOperandPredicate: { 320 uint64_t InsnID = readULEB(); 321 unsigned OpIdx = 322 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1; 323 uint16_t Predicate = readU16(); 324 DEBUG_WITH_TYPE(TgtExecutor::getName(), 325 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" 326 << InsnID << "]->getOperand(" << OpIdx 327 << "), Predicate=" << Predicate << ")\n"); 328 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 329 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || 330 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && 331 "Expected immediate operand"); 332 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 333 int64_t Value = 0; 334 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) 335 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); 336 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) 337 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); 338 else 339 llvm_unreachable("Expected Imm or CImm operand"); 340 341 if (!testImmPredicate_I64(Predicate, Value)) 342 if (handleReject() == RejectAndGiveUp) 343 return false; 344 break; 345 } 346 case GIM_CheckAPIntImmPredicate: { 347 uint64_t InsnID = readULEB(); 348 uint16_t Predicate = readU16(); 349 DEBUG_WITH_TYPE(TgtExecutor::getName(), 350 dbgs() 351 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" 352 << InsnID << "], Predicate=" << Predicate << ")\n"); 353 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 354 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 355 "Expected G_CONSTANT"); 356 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 357 if (!State.MIs[InsnID]->getOperand(1).isCImm()) 358 llvm_unreachable("Expected Imm or CImm operand"); 359 360 const APInt &Value = 361 State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); 362 if (!testImmPredicate_APInt(Predicate, Value)) 363 if (handleReject() == RejectAndGiveUp) 364 return false; 365 break; 366 } 367 case GIM_CheckAPFloatImmPredicate: { 368 uint64_t InsnID = readULEB(); 369 uint16_t Predicate = readU16(); 370 DEBUG_WITH_TYPE(TgtExecutor::getName(), 371 dbgs() 372 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" 373 << InsnID << "], Predicate=" << Predicate << ")\n"); 374 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 375 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 376 "Expected G_FCONSTANT"); 377 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && 378 "Expected FPImm operand"); 379 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 380 const APFloat &Value = 381 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); 382 383 if (!testImmPredicate_APFloat(Predicate, Value)) 384 if (handleReject() == RejectAndGiveUp) 385 return false; 386 break; 387 } 388 case GIM_CheckIsBuildVectorAllOnes: 389 case GIM_CheckIsBuildVectorAllZeros: { 390 uint64_t InsnID = readULEB(); 391 392 DEBUG_WITH_TYPE(TgtExecutor::getName(), 393 dbgs() << CurrentIdx 394 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" 395 << InsnID << "])\n"); 396 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 397 398 const MachineInstr *MI = State.MIs[InsnID]; 399 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || 400 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && 401 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); 402 403 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { 404 if (!isBuildVectorAllOnes(*MI, MRI)) { 405 if (handleReject() == RejectAndGiveUp) 406 return false; 407 } 408 } else { 409 if (!isBuildVectorAllZeros(*MI, MRI)) { 410 if (handleReject() == RejectAndGiveUp) 411 return false; 412 } 413 } 414 415 break; 416 } 417 case GIM_CheckSimplePredicate: { 418 // Note: we don't check for invalid here because this is purely a hook to 419 // allow some executors (such as the combiner) to check arbitrary, 420 // contextless predicates, such as whether a rule is enabled or not. 421 uint16_t Predicate = readU16(); 422 DEBUG_WITH_TYPE(TgtExecutor::getName(), 423 dbgs() << CurrentIdx 424 << ": GIM_CheckSimplePredicate(Predicate=" 425 << Predicate << ")\n"); 426 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 427 if (!testSimplePredicate(Predicate)) { 428 if (handleReject() == RejectAndGiveUp) 429 return false; 430 } 431 break; 432 } 433 case GIM_CheckCxxInsnPredicate: { 434 uint64_t InsnID = readULEB(); 435 uint16_t Predicate = readU16(); 436 DEBUG_WITH_TYPE(TgtExecutor::getName(), 437 dbgs() 438 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" 439 << InsnID << "], Predicate=" << Predicate << ")\n"); 440 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 441 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 442 443 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) 444 if (handleReject() == RejectAndGiveUp) 445 return false; 446 break; 447 } 448 case GIM_CheckHasNoUse: { 449 uint64_t InsnID = readULEB(); 450 451 DEBUG_WITH_TYPE(TgtExecutor::getName(), 452 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" 453 << InsnID << "]\n"); 454 455 const MachineInstr *MI = State.MIs[InsnID]; 456 assert(MI && "Used insn before defined"); 457 assert(MI->getNumDefs() > 0 && "No defs"); 458 const Register Res = MI->getOperand(0).getReg(); 459 460 if (!MRI.use_nodbg_empty(Res)) { 461 if (handleReject() == RejectAndGiveUp) 462 return false; 463 } 464 465 break; 466 } 467 case GIM_CheckAtomicOrdering: { 468 uint64_t InsnID = readULEB(); 469 auto Ordering = (AtomicOrdering)readULEB(); 470 DEBUG_WITH_TYPE(TgtExecutor::getName(), 471 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" 472 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 473 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 474 if (!State.MIs[InsnID]->hasOneMemOperand()) 475 if (handleReject() == RejectAndGiveUp) 476 return false; 477 478 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 479 if (MMO->getMergedOrdering() != Ordering) 480 if (handleReject() == RejectAndGiveUp) 481 return false; 482 break; 483 } 484 case GIM_CheckAtomicOrderingOrStrongerThan: { 485 uint64_t InsnID = readULEB(); 486 auto Ordering = (AtomicOrdering)readULEB(); 487 DEBUG_WITH_TYPE(TgtExecutor::getName(), 488 dbgs() << CurrentIdx 489 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" 490 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 491 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 492 if (!State.MIs[InsnID]->hasOneMemOperand()) 493 if (handleReject() == RejectAndGiveUp) 494 return false; 495 496 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 497 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) 498 if (handleReject() == RejectAndGiveUp) 499 return false; 500 break; 501 } 502 case GIM_CheckAtomicOrderingWeakerThan: { 503 uint64_t InsnID = readULEB(); 504 auto Ordering = (AtomicOrdering)readULEB(); 505 DEBUG_WITH_TYPE(TgtExecutor::getName(), 506 dbgs() << CurrentIdx 507 << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" 508 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 509 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 510 if (!State.MIs[InsnID]->hasOneMemOperand()) 511 if (handleReject() == RejectAndGiveUp) 512 return false; 513 514 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 515 if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) 516 if (handleReject() == RejectAndGiveUp) 517 return false; 518 break; 519 } 520 case GIM_CheckMemoryAddressSpace: { 521 uint64_t InsnID = readULEB(); 522 uint64_t MMOIdx = readULEB(); 523 // This accepts a list of possible address spaces. 524 const uint64_t NumAddrSpace = readULEB(); 525 526 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 527 if (handleReject() == RejectAndGiveUp) 528 return false; 529 break; 530 } 531 532 // Need to still jump to the end of the list of address spaces if we find 533 // a match earlier. 534 const uint64_t LastIdx = CurrentIdx + NumAddrSpace; 535 536 const MachineMemOperand *MMO = 537 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 538 const unsigned MMOAddrSpace = MMO->getAddrSpace(); 539 540 bool Success = false; 541 for (unsigned I = 0; I != NumAddrSpace; ++I) { 542 uint64_t AddrSpace = readULEB(); 543 DEBUG_WITH_TYPE(TgtExecutor::getName(), 544 dbgs() << "addrspace(" << MMOAddrSpace << ") vs " 545 << AddrSpace << '\n'); 546 547 if (AddrSpace == MMOAddrSpace) { 548 Success = true; 549 break; 550 } 551 } 552 553 CurrentIdx = LastIdx; 554 if (!Success && handleReject() == RejectAndGiveUp) 555 return false; 556 break; 557 } 558 case GIM_CheckMemoryAlignment: { 559 uint64_t InsnID = readULEB(); 560 uint64_t MMOIdx = readULEB(); 561 uint64_t MinAlign = readULEB(); 562 563 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 564 565 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 566 if (handleReject() == RejectAndGiveUp) 567 return false; 568 break; 569 } 570 571 MachineMemOperand *MMO = 572 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 573 DEBUG_WITH_TYPE(TgtExecutor::getName(), 574 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" 575 << "(MIs[" << InsnID << "]->memoperands() + " 576 << MMOIdx << ")->getAlignment() >= " << MinAlign 577 << ")\n"); 578 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) 579 return false; 580 581 break; 582 } 583 case GIM_CheckMemorySizeEqualTo: { 584 uint64_t InsnID = readULEB(); 585 uint64_t MMOIdx = readULEB(); 586 uint32_t Size = readU32(); 587 588 DEBUG_WITH_TYPE(TgtExecutor::getName(), 589 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" 590 << InsnID << "]->memoperands() + " << MMOIdx 591 << ", Size=" << Size << ")\n"); 592 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 593 594 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 595 if (handleReject() == RejectAndGiveUp) 596 return false; 597 break; 598 } 599 600 MachineMemOperand *MMO = 601 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 602 603 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() 604 << " bytes vs " << Size 605 << " bytes\n"); 606 if (MMO->getSize() != Size) 607 if (handleReject() == RejectAndGiveUp) 608 return false; 609 610 break; 611 } 612 case GIM_CheckMemorySizeEqualToLLT: 613 case GIM_CheckMemorySizeLessThanLLT: 614 case GIM_CheckMemorySizeGreaterThanLLT: { 615 uint64_t InsnID = readULEB(); 616 uint64_t MMOIdx = readULEB(); 617 uint64_t OpIdx = readULEB(); 618 619 DEBUG_WITH_TYPE( 620 TgtExecutor::getName(), 621 dbgs() << CurrentIdx << ": GIM_CheckMemorySize" 622 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" 623 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT 624 ? "GreaterThan" 625 : "LessThan") 626 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 627 << ", OpIdx=" << OpIdx << ")\n"); 628 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 629 630 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 631 if (!MO.isReg()) { 632 DEBUG_WITH_TYPE(TgtExecutor::getName(), 633 dbgs() << CurrentIdx << ": Not a register\n"); 634 if (handleReject() == RejectAndGiveUp) 635 return false; 636 break; 637 } 638 639 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 640 if (handleReject() == RejectAndGiveUp) 641 return false; 642 break; 643 } 644 645 MachineMemOperand *MMO = 646 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 647 648 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); 649 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && 650 MMO->getSizeInBits() != Size) { 651 if (handleReject() == RejectAndGiveUp) 652 return false; 653 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && 654 MMO->getSizeInBits() >= Size) { 655 if (handleReject() == RejectAndGiveUp) 656 return false; 657 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && 658 MMO->getSizeInBits() <= Size) 659 if (handleReject() == RejectAndGiveUp) 660 return false; 661 662 break; 663 } 664 case GIM_CheckType: { 665 uint64_t InsnID = readULEB(); 666 uint64_t OpIdx = readULEB(); 667 int TypeID = readS8(); 668 DEBUG_WITH_TYPE(TgtExecutor::getName(), 669 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID 670 << "]->getOperand(" << OpIdx 671 << "), TypeID=" << TypeID << ")\n"); 672 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 673 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 674 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) { 675 if (handleReject() == RejectAndGiveUp) 676 return false; 677 } 678 break; 679 } 680 case GIM_CheckPointerToAny: { 681 uint64_t InsnID = readULEB(); 682 uint64_t OpIdx = readULEB(); 683 uint64_t SizeInBits = readULEB(); 684 685 DEBUG_WITH_TYPE(TgtExecutor::getName(), 686 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" 687 << InsnID << "]->getOperand(" << OpIdx 688 << "), SizeInBits=" << SizeInBits << ")\n"); 689 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 690 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 691 const LLT Ty = MRI.getType(MO.getReg()); 692 693 // iPTR must be looked up in the target. 694 if (SizeInBits == 0) { 695 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); 696 const unsigned AddrSpace = Ty.getAddressSpace(); 697 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); 698 } 699 700 assert(SizeInBits != 0 && "Pointer size must be known"); 701 702 if (MO.isReg()) { 703 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) 704 if (handleReject() == RejectAndGiveUp) 705 return false; 706 } else if (handleReject() == RejectAndGiveUp) 707 return false; 708 709 break; 710 } 711 case GIM_RecordNamedOperand: { 712 uint64_t InsnID = readULEB(); 713 uint64_t OpIdx = readULEB(); 714 uint64_t StoreIdx = readULEB(); 715 716 DEBUG_WITH_TYPE(TgtExecutor::getName(), 717 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" 718 << InsnID << "]->getOperand(" << OpIdx 719 << "), StoreIdx=" << StoreIdx << ")\n"); 720 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 721 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); 722 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); 723 break; 724 } 725 case GIM_RecordRegType: { 726 uint64_t InsnID = readULEB(); 727 uint64_t OpIdx = readULEB(); 728 int TypeIdx = readS8(); 729 730 DEBUG_WITH_TYPE(TgtExecutor::getName(), 731 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs[" 732 << InsnID << "]->getOperand(" << OpIdx 733 << "), TypeIdx=" << TypeIdx << ")\n"); 734 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 735 assert(TypeIdx < 0 && "Temp types always have negative indexes!"); 736 // Indexes start at -1. 737 TypeIdx = 1 - TypeIdx; 738 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx); 739 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx) 740 State.RecordedTypes.resize(TypeIdx + 1, LLT()); 741 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg()); 742 break; 743 } 744 case GIM_CheckRegBankForClass: { 745 uint64_t InsnID = readULEB(); 746 uint64_t OpIdx = readULEB(); 747 uint16_t RCEnum = readU16(); 748 DEBUG_WITH_TYPE(TgtExecutor::getName(), 749 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" 750 << InsnID << "]->getOperand(" << OpIdx 751 << "), RCEnum=" << RCEnum << ")\n"); 752 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 753 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 754 if (!MO.isReg() || 755 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), 756 MRI.getType(MO.getReg())) != 757 RBI.getRegBank(MO.getReg(), MRI, TRI)) { 758 if (handleReject() == RejectAndGiveUp) 759 return false; 760 } 761 break; 762 } 763 764 case GIM_CheckComplexPattern: { 765 uint64_t InsnID = readULEB(); 766 uint64_t OpIdx = readULEB(); 767 uint16_t RendererID = readU16(); 768 uint16_t ComplexPredicateID = readU16(); 769 DEBUG_WITH_TYPE(TgtExecutor::getName(), 770 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID 771 << "] = GIM_CheckComplexPattern(MIs[" << InsnID 772 << "]->getOperand(" << OpIdx 773 << "), ComplexPredicateID=" << ComplexPredicateID 774 << ")\n"); 775 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 776 // FIXME: Use std::invoke() when it's available. 777 ComplexRendererFns Renderer = 778 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( 779 State.MIs[InsnID]->getOperand(OpIdx)); 780 if (Renderer) 781 State.Renderers[RendererID] = *Renderer; 782 else if (handleReject() == RejectAndGiveUp) 783 return false; 784 break; 785 } 786 787 case GIM_CheckConstantInt: 788 case GIM_CheckConstantInt8: { 789 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8); 790 791 uint64_t InsnID = readULEB(); 792 uint64_t OpIdx = readULEB(); 793 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64(); 794 DEBUG_WITH_TYPE(TgtExecutor::getName(), 795 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" 796 << InsnID << "]->getOperand(" << OpIdx 797 << "), Value=" << Value << ")\n"); 798 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 799 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 800 if (MO.isReg()) { 801 // isOperandImmEqual() will sign-extend to 64-bits, so should we. 802 LLT Ty = MRI.getType(MO.getReg()); 803 // If the type is > 64 bits, it can't be a constant int, so we bail 804 // early because SignExtend64 will assert otherwise. 805 if (Ty.getScalarSizeInBits() > 64) { 806 if (handleReject() == RejectAndGiveUp) 807 return false; 808 break; 809 } 810 811 Value = SignExtend64(Value, Ty.getScalarSizeInBits()); 812 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) { 813 if (handleReject() == RejectAndGiveUp) 814 return false; 815 } 816 } else if (handleReject() == RejectAndGiveUp) 817 return false; 818 819 break; 820 } 821 822 case GIM_CheckLiteralInt: { 823 uint64_t InsnID = readULEB(); 824 uint64_t OpIdx = readULEB(); 825 int64_t Value = readU64(); 826 DEBUG_WITH_TYPE(TgtExecutor::getName(), 827 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" 828 << InsnID << "]->getOperand(" << OpIdx 829 << "), Value=" << Value << ")\n"); 830 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 831 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 832 if (MO.isImm() && MO.getImm() == Value) 833 break; 834 835 if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) 836 break; 837 838 if (handleReject() == RejectAndGiveUp) 839 return false; 840 841 break; 842 } 843 844 case GIM_CheckIntrinsicID: { 845 uint64_t InsnID = readULEB(); 846 uint64_t OpIdx = readULEB(); 847 uint16_t Value = readU16(); 848 DEBUG_WITH_TYPE(TgtExecutor::getName(), 849 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" 850 << InsnID << "]->getOperand(" << OpIdx 851 << "), Value=" << Value << ")\n"); 852 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 853 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 854 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) 855 if (handleReject() == RejectAndGiveUp) 856 return false; 857 break; 858 } 859 case GIM_CheckCmpPredicate: { 860 uint64_t InsnID = readULEB(); 861 uint64_t OpIdx = readULEB(); 862 uint16_t Value = readU16(); 863 DEBUG_WITH_TYPE(TgtExecutor::getName(), 864 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" 865 << InsnID << "]->getOperand(" << OpIdx 866 << "), Value=" << Value << ")\n"); 867 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 868 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 869 if (!MO.isPredicate() || MO.getPredicate() != Value) 870 if (handleReject() == RejectAndGiveUp) 871 return false; 872 break; 873 } 874 case GIM_CheckIsMBB: { 875 uint64_t InsnID = readULEB(); 876 uint64_t OpIdx = readULEB(); 877 DEBUG_WITH_TYPE(TgtExecutor::getName(), 878 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID 879 << "]->getOperand(" << OpIdx << "))\n"); 880 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 881 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { 882 if (handleReject() == RejectAndGiveUp) 883 return false; 884 } 885 break; 886 } 887 case GIM_CheckIsImm: { 888 uint64_t InsnID = readULEB(); 889 uint64_t OpIdx = readULEB(); 890 DEBUG_WITH_TYPE(TgtExecutor::getName(), 891 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID 892 << "]->getOperand(" << OpIdx << "))\n"); 893 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 894 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { 895 if (handleReject() == RejectAndGiveUp) 896 return false; 897 } 898 break; 899 } 900 case GIM_CheckIsSafeToFold: { 901 uint64_t InsnID = readULEB(); 902 DEBUG_WITH_TYPE(TgtExecutor::getName(), 903 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" 904 << InsnID << "])\n"); 905 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 906 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { 907 if (handleReject() == RejectAndGiveUp) 908 return false; 909 } 910 break; 911 } 912 case GIM_CheckIsSameOperand: 913 case GIM_CheckIsSameOperandIgnoreCopies: { 914 uint64_t InsnID = readULEB(); 915 uint64_t OpIdx = readULEB(); 916 uint64_t OtherInsnID = readULEB(); 917 uint64_t OtherOpIdx = readULEB(); 918 DEBUG_WITH_TYPE(TgtExecutor::getName(), 919 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" 920 << InsnID << "][" << OpIdx << "], MIs[" 921 << OtherInsnID << "][" << OtherOpIdx << "])\n"); 922 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 923 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); 924 925 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); 926 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); 927 928 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { 929 if (Op.isReg() && OtherOp.isReg()) { 930 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) == 931 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI)) 932 break; 933 } 934 } 935 936 if (!Op.isIdenticalTo(OtherOp)) { 937 if (handleReject() == RejectAndGiveUp) 938 return false; 939 } 940 break; 941 } 942 case GIM_CheckCanReplaceReg: { 943 uint64_t OldInsnID = readULEB(); 944 uint64_t OldOpIdx = readULEB(); 945 uint64_t NewInsnID = readULEB(); 946 uint64_t NewOpIdx = readULEB(); 947 948 DEBUG_WITH_TYPE(TgtExecutor::getName(), 949 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs[" 950 << OldInsnID << "][" << OldOpIdx << "] = MIs[" 951 << NewInsnID << "][" << NewOpIdx << "])\n"); 952 953 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 954 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); 955 if (!canReplaceReg(Old, New, MRI)) { 956 if (handleReject() == RejectAndGiveUp) 957 return false; 958 } 959 break; 960 } 961 case GIM_MIFlags: { 962 uint64_t InsnID = readULEB(); 963 uint32_t Flags = readU32(); 964 965 DEBUG_WITH_TYPE(TgtExecutor::getName(), 966 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID 967 << "], " << Flags << ")\n"); 968 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) { 969 if (handleReject() == RejectAndGiveUp) 970 return false; 971 } 972 break; 973 } 974 case GIM_MIFlagsNot: { 975 uint64_t InsnID = readULEB(); 976 uint32_t Flags = readU32(); 977 978 DEBUG_WITH_TYPE(TgtExecutor::getName(), 979 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID 980 << "], " << Flags << ")\n"); 981 if ((State.MIs[InsnID]->getFlags() & Flags)) { 982 if (handleReject() == RejectAndGiveUp) 983 return false; 984 } 985 break; 986 } 987 case GIM_Reject: 988 DEBUG_WITH_TYPE(TgtExecutor::getName(), 989 dbgs() << CurrentIdx << ": GIM_Reject\n"); 990 if (handleReject() == RejectAndGiveUp) 991 return false; 992 break; 993 case GIR_MutateOpcode: { 994 uint64_t OldInsnID = readULEB(); 995 uint64_t NewInsnID = readULEB(); 996 uint16_t NewOpcode = readU16(); 997 if (NewInsnID >= OutMIs.size()) 998 OutMIs.resize(NewInsnID + 1); 999 1000 MachineInstr *OldMI = State.MIs[OldInsnID]; 1001 if (Observer) 1002 Observer->changingInstr(*OldMI); 1003 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI); 1004 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); 1005 if (Observer) 1006 Observer->changedInstr(*OldMI); 1007 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1008 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" 1009 << NewInsnID << "], MIs[" << OldInsnID << "], " 1010 << NewOpcode << ")\n"); 1011 break; 1012 } 1013 1014 case GIR_BuildMI: { 1015 uint64_t NewInsnID = readULEB(); 1016 uint16_t Opcode = readU16(); 1017 if (NewInsnID >= OutMIs.size()) 1018 OutMIs.resize(NewInsnID + 1); 1019 1020 OutMIs[NewInsnID] = Builder.buildInstr(Opcode); 1021 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1022 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" 1023 << NewInsnID << "], " << Opcode << ")\n"); 1024 break; 1025 } 1026 1027 case GIR_BuildConstant: { 1028 uint64_t TempRegID = readULEB(); 1029 uint64_t Imm = readU64(); 1030 Builder.buildConstant(State.TempRegisters[TempRegID], Imm); 1031 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1032 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg[" 1033 << TempRegID << "], Imm=" << Imm << ")\n"); 1034 break; 1035 } 1036 1037 case GIR_Copy: { 1038 uint64_t NewInsnID = readULEB(); 1039 uint64_t OldInsnID = readULEB(); 1040 uint64_t OpIdx = readULEB(); 1041 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1042 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); 1043 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1044 dbgs() 1045 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID 1046 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); 1047 break; 1048 } 1049 1050 case GIR_CopyOrAddZeroReg: { 1051 uint64_t NewInsnID = readULEB(); 1052 uint64_t OldInsnID = readULEB(); 1053 uint64_t OpIdx = readULEB(); 1054 uint16_t ZeroReg = readU16(); 1055 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1056 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); 1057 if (isOperandImmEqual(MO, 0, MRI)) 1058 OutMIs[NewInsnID].addReg(ZeroReg); 1059 else 1060 OutMIs[NewInsnID].add(MO); 1061 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1062 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" 1063 << NewInsnID << "], MIs[" << OldInsnID << "], " 1064 << OpIdx << ", " << ZeroReg << ")\n"); 1065 break; 1066 } 1067 1068 case GIR_CopySubReg: { 1069 uint64_t NewInsnID = readULEB(); 1070 uint64_t OldInsnID = readULEB(); 1071 uint64_t OpIdx = readULEB(); 1072 uint16_t SubRegIdx = readU16(); 1073 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1074 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 1075 0, SubRegIdx); 1076 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1077 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" 1078 << NewInsnID << "], MIs[" << OldInsnID << "], " 1079 << OpIdx << ", " << SubRegIdx << ")\n"); 1080 break; 1081 } 1082 1083 case GIR_AddImplicitDef: { 1084 uint64_t InsnID = readULEB(); 1085 uint16_t RegNum = readU16(); 1086 uint16_t Flags = readU16(); 1087 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1088 Flags |= RegState::Implicit; 1089 OutMIs[InsnID].addDef(RegNum, Flags); 1090 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1091 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" 1092 << InsnID << "], " << RegNum << ")\n"); 1093 break; 1094 } 1095 1096 case GIR_AddImplicitUse: { 1097 uint64_t InsnID = readULEB(); 1098 uint16_t RegNum = readU16(); 1099 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1100 OutMIs[InsnID].addUse(RegNum, RegState::Implicit); 1101 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1102 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" 1103 << InsnID << "], " << RegNum << ")\n"); 1104 break; 1105 } 1106 1107 case GIR_AddRegister: { 1108 uint64_t InsnID = readULEB(); 1109 uint16_t RegNum = readU16(); 1110 uint16_t RegFlags = readU16(); 1111 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1112 OutMIs[InsnID].addReg(RegNum, RegFlags); 1113 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1114 dbgs() 1115 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID 1116 << "], " << RegNum << ", " << RegFlags << ")\n"); 1117 break; 1118 } 1119 case GIR_SetImplicitDefDead: { 1120 uint64_t InsnID = readULEB(); 1121 uint64_t OpIdx = readULEB(); 1122 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1123 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs[" 1124 << InsnID << "], OpIdx=" << OpIdx << ")\n"); 1125 MachineInstr *MI = OutMIs[InsnID]; 1126 assert(MI && "Modifying undefined instruction"); 1127 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead(); 1128 break; 1129 } 1130 case GIR_SetMIFlags: { 1131 uint64_t InsnID = readULEB(); 1132 uint32_t Flags = readU32(); 1133 1134 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1135 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs[" 1136 << InsnID << "], " << Flags << ")\n"); 1137 MachineInstr *MI = OutMIs[InsnID]; 1138 MI->setFlags(MI->getFlags() | Flags); 1139 break; 1140 } 1141 case GIR_UnsetMIFlags: { 1142 uint64_t InsnID = readULEB(); 1143 uint32_t Flags = readU32(); 1144 1145 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1146 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs[" 1147 << InsnID << "], " << Flags << ")\n"); 1148 MachineInstr *MI = OutMIs[InsnID]; 1149 MI->setFlags(MI->getFlags() & ~Flags); 1150 break; 1151 } 1152 case GIR_CopyMIFlags: { 1153 uint64_t InsnID = readULEB(); 1154 uint64_t OldInsnID = readULEB(); 1155 1156 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1157 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs[" 1158 << InsnID << "], MIs[" << OldInsnID << "])\n"); 1159 MachineInstr *MI = OutMIs[InsnID]; 1160 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags()); 1161 break; 1162 } 1163 case GIR_AddSimpleTempRegister: 1164 case GIR_AddTempRegister: 1165 case GIR_AddTempSubRegister: { 1166 uint64_t InsnID = readULEB(); 1167 uint64_t TempRegID = readULEB(); 1168 uint16_t TempRegFlags = 0; 1169 if (MatcherOpcode != GIR_AddSimpleTempRegister) 1170 TempRegFlags = readU16(); 1171 uint16_t SubReg = 0; 1172 if (MatcherOpcode == GIR_AddTempSubRegister) 1173 SubReg = readU16(); 1174 1175 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1176 1177 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, 1178 SubReg); 1179 DEBUG_WITH_TYPE( 1180 TgtExecutor::getName(), 1181 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID 1182 << "], TempRegisters[" << TempRegID << "]"; 1183 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); 1184 dbgs() << ", " << TempRegFlags << ")\n"); 1185 break; 1186 } 1187 1188 case GIR_AddImm8: 1189 case GIR_AddImm: { 1190 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8); 1191 uint64_t InsnID = readULEB(); 1192 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64(); 1193 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1194 OutMIs[InsnID].addImm(Imm); 1195 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1196 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID 1197 << "], " << Imm << ")\n"); 1198 break; 1199 } 1200 1201 case GIR_AddCImm: { 1202 uint64_t InsnID = readULEB(); 1203 int TypeID = readS8(); 1204 uint64_t Imm = readU64(); 1205 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1206 1207 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits(); 1208 LLVMContext &Ctx = MF->getFunction().getContext(); 1209 OutMIs[InsnID].addCImm( 1210 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true)); 1211 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1212 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID 1213 << "], TypeID=" << TypeID << ", Imm=" << Imm 1214 << ")\n"); 1215 break; 1216 } 1217 1218 case GIR_ComplexRenderer: { 1219 uint64_t InsnID = readULEB(); 1220 uint16_t RendererID = readU16(); 1221 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1222 for (const auto &RenderOpFn : State.Renderers[RendererID]) 1223 RenderOpFn(OutMIs[InsnID]); 1224 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1225 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" 1226 << InsnID << "], " << RendererID << ")\n"); 1227 break; 1228 } 1229 case GIR_ComplexSubOperandRenderer: { 1230 uint64_t InsnID = readULEB(); 1231 uint16_t RendererID = readU16(); 1232 uint64_t RenderOpID = readULEB(); 1233 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1234 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); 1235 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1236 dbgs() << CurrentIdx 1237 << ": GIR_ComplexSubOperandRenderer(OutMIs[" 1238 << InsnID << "], " << RendererID << ", " 1239 << RenderOpID << ")\n"); 1240 break; 1241 } 1242 case GIR_ComplexSubOperandSubRegRenderer: { 1243 uint64_t InsnID = readULEB(); 1244 uint16_t RendererID = readU16(); 1245 uint64_t RenderOpID = readULEB(); 1246 uint16_t SubRegIdx = readU16(); 1247 MachineInstrBuilder &MI = OutMIs[InsnID]; 1248 assert(MI && "Attempted to add to undefined instruction"); 1249 State.Renderers[RendererID][RenderOpID](MI); 1250 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx); 1251 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1252 dbgs() << CurrentIdx 1253 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" 1254 << InsnID << "], " << RendererID << ", " 1255 << RenderOpID << ", " << SubRegIdx << ")\n"); 1256 break; 1257 } 1258 1259 case GIR_CopyConstantAsSImm: { 1260 uint64_t NewInsnID = readULEB(); 1261 uint64_t OldInsnID = readULEB(); 1262 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1263 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 1264 "Expected G_CONSTANT"); 1265 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { 1266 OutMIs[NewInsnID].addImm( 1267 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); 1268 } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) 1269 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); 1270 else 1271 llvm_unreachable("Expected Imm or CImm operand"); 1272 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1273 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" 1274 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1275 break; 1276 } 1277 1278 // TODO: Needs a test case once we have a pattern that uses this. 1279 case GIR_CopyFConstantAsFPImm: { 1280 uint64_t NewInsnID = readULEB(); 1281 uint64_t OldInsnID = readULEB(); 1282 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1283 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 1284 "Expected G_FCONSTANT"); 1285 if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) 1286 OutMIs[NewInsnID].addFPImm( 1287 State.MIs[OldInsnID]->getOperand(1).getFPImm()); 1288 else 1289 llvm_unreachable("Expected FPImm operand"); 1290 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1291 dbgs() 1292 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" 1293 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1294 break; 1295 } 1296 1297 case GIR_CustomRenderer: { 1298 uint64_t InsnID = readULEB(); 1299 uint64_t OldInsnID = readULEB(); 1300 uint16_t RendererFnID = readU16(); 1301 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1302 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1303 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" 1304 << InsnID << "], MIs[" << OldInsnID << "], " 1305 << RendererFnID << ")\n"); 1306 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1307 OutMIs[InsnID], *State.MIs[OldInsnID], 1308 -1); // Not a source operand of the old instruction. 1309 break; 1310 } 1311 case GIR_CustomAction: { 1312 uint16_t FnID = readU16(); 1313 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1314 dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID 1315 << ")\n"); 1316 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID"); 1317 runCustomAction(FnID, State, OutMIs); 1318 break; 1319 } 1320 case GIR_CustomOperandRenderer: { 1321 uint64_t InsnID = readULEB(); 1322 uint64_t OldInsnID = readULEB(); 1323 uint64_t OpIdx = readULEB(); 1324 uint16_t RendererFnID = readU16(); 1325 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1326 1327 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1328 dbgs() << CurrentIdx 1329 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID 1330 << "], MIs[" << OldInsnID << "]->getOperand(" 1331 << OpIdx << "), " << RendererFnID << ")\n"); 1332 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1333 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); 1334 break; 1335 } 1336 case GIR_ConstrainOperandRC: { 1337 uint64_t InsnID = readULEB(); 1338 uint64_t OpIdx = readULEB(); 1339 uint16_t RCEnum = readU16(); 1340 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1341 MachineInstr &I = *OutMIs[InsnID].getInstr(); 1342 MachineFunction &MF = *I.getParent()->getParent(); 1343 MachineRegisterInfo &MRI = MF.getRegInfo(); 1344 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); 1345 MachineOperand &MO = I.getOperand(OpIdx); 1346 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); 1347 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1348 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" 1349 << InsnID << "], " << OpIdx << ", " << RCEnum 1350 << ")\n"); 1351 break; 1352 } 1353 1354 case GIR_ConstrainSelectedInstOperands: { 1355 uint64_t InsnID = readULEB(); 1356 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1357 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, 1358 RBI); 1359 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1360 dbgs() << CurrentIdx 1361 << ": GIR_ConstrainSelectedInstOperands(OutMIs[" 1362 << InsnID << "])\n"); 1363 break; 1364 } 1365 1366 case GIR_MergeMemOperands: { 1367 uint64_t InsnID = readULEB(); 1368 uint64_t NumInsn = MatchTable[CurrentIdx++]; 1369 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1370 1371 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1372 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" 1373 << InsnID << "]"); 1374 for (unsigned K = 0; K < NumInsn; ++K) { 1375 uint64_t NextID = readULEB(); 1376 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1377 dbgs() << ", MIs[" << NextID << "]"); 1378 for (const auto &MMO : State.MIs[NextID]->memoperands()) 1379 OutMIs[InsnID].addMemOperand(MMO); 1380 } 1381 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n"); 1382 break; 1383 } 1384 1385 case GIR_EraseFromParent: { 1386 uint64_t InsnID = readULEB(); 1387 MachineInstr *MI = State.MIs[InsnID]; 1388 assert(MI && "Attempted to erase an undefined instruction"); 1389 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1390 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" 1391 << InsnID << "])\n"); 1392 // If we're erasing the insertion point, ensure we don't leave a dangling 1393 // pointer in the builder. 1394 if (Builder.getInsertPt() == MI) 1395 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator()); 1396 if (Observer) 1397 Observer->erasingInstr(*MI); 1398 MI->eraseFromParent(); 1399 break; 1400 } 1401 1402 case GIR_MakeTempReg: { 1403 uint64_t TempRegID = readULEB(); 1404 int TypeID = readS8(); 1405 1406 State.TempRegisters[TempRegID] = 1407 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID)); 1408 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1409 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID 1410 << "] = GIR_MakeTempReg(" << TypeID << ")\n"); 1411 break; 1412 } 1413 case GIR_ReplaceReg: { 1414 uint64_t OldInsnID = readULEB(); 1415 uint64_t OldOpIdx = readULEB(); 1416 uint64_t NewInsnID = readULEB(); 1417 uint64_t NewOpIdx = readULEB(); 1418 1419 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1420 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs[" 1421 << OldInsnID << "][" << OldOpIdx << "] = MIs[" 1422 << NewInsnID << "][" << NewOpIdx << "])\n"); 1423 1424 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 1425 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); 1426 if (Observer) 1427 Observer->changingAllUsesOfReg(MRI, Old); 1428 MRI.replaceRegWith(Old, New); 1429 if (Observer) 1430 Observer->finishedChangingAllUsesOfReg(); 1431 break; 1432 } 1433 case GIR_ReplaceRegWithTempReg: { 1434 uint64_t OldInsnID = readULEB(); 1435 uint64_t OldOpIdx = readULEB(); 1436 uint64_t TempRegID = readULEB(); 1437 1438 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1439 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs[" 1440 << OldInsnID << "][" << OldOpIdx << "] = TempRegs[" 1441 << TempRegID << "])\n"); 1442 1443 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 1444 Register New = State.TempRegisters[TempRegID]; 1445 if (Observer) 1446 Observer->changingAllUsesOfReg(MRI, Old); 1447 MRI.replaceRegWith(Old, New); 1448 if (Observer) 1449 Observer->finishedChangingAllUsesOfReg(); 1450 break; 1451 } 1452 case GIR_Coverage: { 1453 uint32_t RuleID = readU32(); 1454 assert(CoverageInfo); 1455 CoverageInfo->setCovered(RuleID); 1456 1457 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx 1458 << ": GIR_Coverage(" 1459 << RuleID << ")"); 1460 break; 1461 } 1462 1463 case GIR_Done: 1464 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1465 dbgs() << CurrentIdx << ": GIR_Done\n"); 1466 propagateFlags(); 1467 return true; 1468 default: 1469 llvm_unreachable("Unexpected command"); 1470 } 1471 } 1472 } 1473 1474 } // end namespace llvm 1475 1476 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 1477