10b57cec5SDimitry Andric //===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // Pass to verify generated machine code. The following is checked:
100b57cec5SDimitry Andric //
110b57cec5SDimitry Andric // Operand counts: All explicit operands must be present.
120b57cec5SDimitry Andric //
130b57cec5SDimitry Andric // Register classes: All physical and virtual register operands must be
140b57cec5SDimitry Andric // compatible with the register class required by the instruction descriptor.
150b57cec5SDimitry Andric //
160b57cec5SDimitry Andric // Register live intervals: Registers must be defined only once, and must be
170b57cec5SDimitry Andric // defined before use.
180b57cec5SDimitry Andric //
195ffd83dbSDimitry Andric // The machine code verifier is enabled with the command-line option
205ffd83dbSDimitry Andric // -verify-machineinstrs.
210b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
220b57cec5SDimitry Andric
230b57cec5SDimitry Andric #include "llvm/ADT/BitVector.h"
240b57cec5SDimitry Andric #include "llvm/ADT/DenseMap.h"
250b57cec5SDimitry Andric #include "llvm/ADT/DenseSet.h"
260b57cec5SDimitry Andric #include "llvm/ADT/DepthFirstIterator.h"
275ffd83dbSDimitry Andric #include "llvm/ADT/PostOrderIterator.h"
280b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
290b57cec5SDimitry Andric #include "llvm/ADT/SetOperations.h"
300b57cec5SDimitry Andric #include "llvm/ADT/SmallPtrSet.h"
310b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
320b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
330b57cec5SDimitry Andric #include "llvm/ADT/Twine.h"
3481ad6265SDimitry Andric #include "llvm/CodeGen/CodeGenCommonISel.h"
355f757f3fSDimitry Andric #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
360b57cec5SDimitry Andric #include "llvm/CodeGen/LiveInterval.h"
370b57cec5SDimitry Andric #include "llvm/CodeGen/LiveIntervals.h"
3881ad6265SDimitry Andric #include "llvm/CodeGen/LiveRangeCalc.h"
390b57cec5SDimitry Andric #include "llvm/CodeGen/LiveStacks.h"
400b57cec5SDimitry Andric #include "llvm/CodeGen/LiveVariables.h"
4106c3fb27SDimitry Andric #include "llvm/CodeGen/LowLevelType.h"
420b57cec5SDimitry Andric #include "llvm/CodeGen/MachineBasicBlock.h"
430b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFrameInfo.h"
440b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h"
450b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunctionPass.h"
460b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstr.h"
470b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstrBundle.h"
480b57cec5SDimitry Andric #include "llvm/CodeGen/MachineMemOperand.h"
490b57cec5SDimitry Andric #include "llvm/CodeGen/MachineOperand.h"
500b57cec5SDimitry Andric #include "llvm/CodeGen/MachineRegisterInfo.h"
510b57cec5SDimitry Andric #include "llvm/CodeGen/PseudoSourceValue.h"
5281ad6265SDimitry Andric #include "llvm/CodeGen/RegisterBank.h"
5381ad6265SDimitry Andric #include "llvm/CodeGen/RegisterBankInfo.h"
540b57cec5SDimitry Andric #include "llvm/CodeGen/SlotIndexes.h"
550b57cec5SDimitry Andric #include "llvm/CodeGen/StackMaps.h"
560b57cec5SDimitry Andric #include "llvm/CodeGen/TargetInstrInfo.h"
570b57cec5SDimitry Andric #include "llvm/CodeGen/TargetOpcodes.h"
580b57cec5SDimitry Andric #include "llvm/CodeGen/TargetRegisterInfo.h"
590b57cec5SDimitry Andric #include "llvm/CodeGen/TargetSubtargetInfo.h"
600b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h"
6181ad6265SDimitry Andric #include "llvm/IR/Constants.h"
6206c3fb27SDimitry Andric #include "llvm/IR/EHPersonalities.h"
630b57cec5SDimitry Andric #include "llvm/IR/Function.h"
640b57cec5SDimitry Andric #include "llvm/IR/InlineAsm.h"
650b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
66480093f4SDimitry Andric #include "llvm/InitializePasses.h"
670b57cec5SDimitry Andric #include "llvm/MC/LaneBitmask.h"
680b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
6981ad6265SDimitry Andric #include "llvm/MC/MCDwarf.h"
700b57cec5SDimitry Andric #include "llvm/MC/MCInstrDesc.h"
710b57cec5SDimitry Andric #include "llvm/MC/MCRegisterInfo.h"
720b57cec5SDimitry Andric #include "llvm/MC/MCTargetOptions.h"
730b57cec5SDimitry Andric #include "llvm/Pass.h"
740b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
750b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
760b57cec5SDimitry Andric #include "llvm/Support/MathExtras.h"
77bdd1243dSDimitry Andric #include "llvm/Support/ModRef.h"
780b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
790b57cec5SDimitry Andric #include "llvm/Target/TargetMachine.h"
800b57cec5SDimitry Andric #include <algorithm>
810b57cec5SDimitry Andric #include <cassert>
820b57cec5SDimitry Andric #include <cstddef>
830b57cec5SDimitry Andric #include <cstdint>
840b57cec5SDimitry Andric #include <iterator>
850b57cec5SDimitry Andric #include <string>
860b57cec5SDimitry Andric #include <utility>
870b57cec5SDimitry Andric
880b57cec5SDimitry Andric using namespace llvm;
890b57cec5SDimitry Andric
900b57cec5SDimitry Andric namespace {
910b57cec5SDimitry Andric
920b57cec5SDimitry Andric struct MachineVerifier {
MachineVerifier__anoncaf807cd0111::MachineVerifier930b57cec5SDimitry Andric MachineVerifier(Pass *pass, const char *b) : PASS(pass), Banner(b) {}
940b57cec5SDimitry Andric
MachineVerifier__anoncaf807cd0111::MachineVerifier955f757f3fSDimitry Andric MachineVerifier(const char *b, LiveVariables *LiveVars,
965f757f3fSDimitry Andric LiveIntervals *LiveInts, LiveStacks *LiveStks,
975f757f3fSDimitry Andric SlotIndexes *Indexes)
985f757f3fSDimitry Andric : Banner(b), LiveVars(LiveVars), LiveInts(LiveInts), LiveStks(LiveStks),
995f757f3fSDimitry Andric Indexes(Indexes) {}
1005f757f3fSDimitry Andric
101e8d8bef9SDimitry Andric unsigned verify(const MachineFunction &MF);
1020b57cec5SDimitry Andric
1035f757f3fSDimitry Andric Pass *const PASS = nullptr;
1040b57cec5SDimitry Andric const char *Banner;
10506c3fb27SDimitry Andric const MachineFunction *MF = nullptr;
10606c3fb27SDimitry Andric const TargetMachine *TM = nullptr;
10706c3fb27SDimitry Andric const TargetInstrInfo *TII = nullptr;
10806c3fb27SDimitry Andric const TargetRegisterInfo *TRI = nullptr;
10906c3fb27SDimitry Andric const MachineRegisterInfo *MRI = nullptr;
11006c3fb27SDimitry Andric const RegisterBankInfo *RBI = nullptr;
1110b57cec5SDimitry Andric
11206c3fb27SDimitry Andric unsigned foundErrors = 0;
1130b57cec5SDimitry Andric
1140b57cec5SDimitry Andric // Avoid querying the MachineFunctionProperties for each operand.
11506c3fb27SDimitry Andric bool isFunctionRegBankSelected = false;
11606c3fb27SDimitry Andric bool isFunctionSelected = false;
11706c3fb27SDimitry Andric bool isFunctionTracksDebugUserValues = false;
1180b57cec5SDimitry Andric
119e8d8bef9SDimitry Andric using RegVector = SmallVector<Register, 16>;
1200b57cec5SDimitry Andric using RegMaskVector = SmallVector<const uint32_t *, 4>;
121e8d8bef9SDimitry Andric using RegSet = DenseSet<Register>;
122e8d8bef9SDimitry Andric using RegMap = DenseMap<Register, const MachineInstr *>;
1230b57cec5SDimitry Andric using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
1240b57cec5SDimitry Andric
12506c3fb27SDimitry Andric const MachineInstr *FirstNonPHI = nullptr;
12606c3fb27SDimitry Andric const MachineInstr *FirstTerminator = nullptr;
1270b57cec5SDimitry Andric BlockSet FunctionBlocks;
1280b57cec5SDimitry Andric
1290b57cec5SDimitry Andric BitVector regsReserved;
1300b57cec5SDimitry Andric RegSet regsLive;
1310b57cec5SDimitry Andric RegVector regsDefined, regsDead, regsKilled;
1320b57cec5SDimitry Andric RegMaskVector regMasks;
1330b57cec5SDimitry Andric
1340b57cec5SDimitry Andric SlotIndex lastIndex;
1350b57cec5SDimitry Andric
1360b57cec5SDimitry Andric // Add Reg and any sub-registers to RV
addRegWithSubRegs__anoncaf807cd0111::MachineVerifier137e8d8bef9SDimitry Andric void addRegWithSubRegs(RegVector &RV, Register Reg) {
1380b57cec5SDimitry Andric RV.push_back(Reg);
139e8d8bef9SDimitry Andric if (Reg.isPhysical())
140e8d8bef9SDimitry Andric append_range(RV, TRI->subregs(Reg.asMCReg()));
1410b57cec5SDimitry Andric }
1420b57cec5SDimitry Andric
1430b57cec5SDimitry Andric struct BBInfo {
1440b57cec5SDimitry Andric // Is this MBB reachable from the MF entry point?
1450b57cec5SDimitry Andric bool reachable = false;
1460b57cec5SDimitry Andric
1470b57cec5SDimitry Andric // Vregs that must be live in because they are used without being
148e8d8bef9SDimitry Andric // defined. Map value is the user. vregsLiveIn doesn't include regs
149e8d8bef9SDimitry Andric // that only are used by PHI nodes.
1500b57cec5SDimitry Andric RegMap vregsLiveIn;
1510b57cec5SDimitry Andric
1520b57cec5SDimitry Andric // Regs killed in MBB. They may be defined again, and will then be in both
1530b57cec5SDimitry Andric // regsKilled and regsLiveOut.
1540b57cec5SDimitry Andric RegSet regsKilled;
1550b57cec5SDimitry Andric
1560b57cec5SDimitry Andric // Regs defined in MBB and live out. Note that vregs passing through may
1570b57cec5SDimitry Andric // be live out without being mentioned here.
1580b57cec5SDimitry Andric RegSet regsLiveOut;
1590b57cec5SDimitry Andric
1600b57cec5SDimitry Andric // Vregs that pass through MBB untouched. This set is disjoint from
1610b57cec5SDimitry Andric // regsKilled and regsLiveOut.
1620b57cec5SDimitry Andric RegSet vregsPassed;
1630b57cec5SDimitry Andric
1640b57cec5SDimitry Andric // Vregs that must pass through MBB because they are needed by a successor
1650b57cec5SDimitry Andric // block. This set is disjoint from regsLiveOut.
1660b57cec5SDimitry Andric RegSet vregsRequired;
1670b57cec5SDimitry Andric
1680b57cec5SDimitry Andric // Set versions of block's predecessor and successor lists.
1690b57cec5SDimitry Andric BlockSet Preds, Succs;
1700b57cec5SDimitry Andric
1710b57cec5SDimitry Andric BBInfo() = default;
1720b57cec5SDimitry Andric
1730b57cec5SDimitry Andric // Add register to vregsRequired if it belongs there. Return true if
1740b57cec5SDimitry Andric // anything changed.
addRequired__anoncaf807cd0111::MachineVerifier::BBInfo175e8d8bef9SDimitry Andric bool addRequired(Register Reg) {
176e8d8bef9SDimitry Andric if (!Reg.isVirtual())
1770b57cec5SDimitry Andric return false;
1780b57cec5SDimitry Andric if (regsLiveOut.count(Reg))
1790b57cec5SDimitry Andric return false;
1800b57cec5SDimitry Andric return vregsRequired.insert(Reg).second;
1810b57cec5SDimitry Andric }
1820b57cec5SDimitry Andric
1830b57cec5SDimitry Andric // Same for a full set.
addRequired__anoncaf807cd0111::MachineVerifier::BBInfo1840b57cec5SDimitry Andric bool addRequired(const RegSet &RS) {
1855ffd83dbSDimitry Andric bool Changed = false;
186e8d8bef9SDimitry Andric for (Register Reg : RS)
1875ffd83dbSDimitry Andric Changed |= addRequired(Reg);
1885ffd83dbSDimitry Andric return Changed;
1890b57cec5SDimitry Andric }
1900b57cec5SDimitry Andric
1910b57cec5SDimitry Andric // Same for a full map.
addRequired__anoncaf807cd0111::MachineVerifier::BBInfo1920b57cec5SDimitry Andric bool addRequired(const RegMap &RM) {
1935ffd83dbSDimitry Andric bool Changed = false;
1945ffd83dbSDimitry Andric for (const auto &I : RM)
1955ffd83dbSDimitry Andric Changed |= addRequired(I.first);
1965ffd83dbSDimitry Andric return Changed;
1970b57cec5SDimitry Andric }
1980b57cec5SDimitry Andric
1990b57cec5SDimitry Andric // Live-out registers are either in regsLiveOut or vregsPassed.
isLiveOut__anoncaf807cd0111::MachineVerifier::BBInfo200e8d8bef9SDimitry Andric bool isLiveOut(Register Reg) const {
2010b57cec5SDimitry Andric return regsLiveOut.count(Reg) || vregsPassed.count(Reg);
2020b57cec5SDimitry Andric }
2030b57cec5SDimitry Andric };
2040b57cec5SDimitry Andric
2050b57cec5SDimitry Andric // Extra register info per MBB.
2060b57cec5SDimitry Andric DenseMap<const MachineBasicBlock*, BBInfo> MBBInfoMap;
2070b57cec5SDimitry Andric
isReserved__anoncaf807cd0111::MachineVerifier208e8d8bef9SDimitry Andric bool isReserved(Register Reg) {
209e8d8bef9SDimitry Andric return Reg.id() < regsReserved.size() && regsReserved.test(Reg.id());
2100b57cec5SDimitry Andric }
2110b57cec5SDimitry Andric
isAllocatable__anoncaf807cd0111::MachineVerifier212e8d8bef9SDimitry Andric bool isAllocatable(Register Reg) const {
213e8d8bef9SDimitry Andric return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) &&
214e8d8bef9SDimitry Andric !regsReserved.test(Reg.id());
2150b57cec5SDimitry Andric }
2160b57cec5SDimitry Andric
2170b57cec5SDimitry Andric // Analysis information if available
21806c3fb27SDimitry Andric LiveVariables *LiveVars = nullptr;
21906c3fb27SDimitry Andric LiveIntervals *LiveInts = nullptr;
22006c3fb27SDimitry Andric LiveStacks *LiveStks = nullptr;
22106c3fb27SDimitry Andric SlotIndexes *Indexes = nullptr;
2220b57cec5SDimitry Andric
2230b57cec5SDimitry Andric void visitMachineFunctionBefore();
2240b57cec5SDimitry Andric void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
2250b57cec5SDimitry Andric void visitMachineBundleBefore(const MachineInstr *MI);
2260b57cec5SDimitry Andric
227349cc55cSDimitry Andric /// Verify that all of \p MI's virtual register operands are scalars.
228349cc55cSDimitry Andric /// \returns True if all virtual register operands are scalar. False
229349cc55cSDimitry Andric /// otherwise.
230349cc55cSDimitry Andric bool verifyAllRegOpsScalar(const MachineInstr &MI,
231349cc55cSDimitry Andric const MachineRegisterInfo &MRI);
2320b57cec5SDimitry Andric bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
2335f757f3fSDimitry Andric
2345f757f3fSDimitry Andric bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
2355f757f3fSDimitry Andric bool verifyGIntrinsicConvergence(const MachineInstr *MI);
2360b57cec5SDimitry Andric void verifyPreISelGenericInstruction(const MachineInstr *MI);
2375f757f3fSDimitry Andric
2380b57cec5SDimitry Andric void visitMachineInstrBefore(const MachineInstr *MI);
2390b57cec5SDimitry Andric void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
2400b57cec5SDimitry Andric void visitMachineBundleAfter(const MachineInstr *MI);
2410b57cec5SDimitry Andric void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
2420b57cec5SDimitry Andric void visitMachineFunctionAfter();
2430b57cec5SDimitry Andric
2440b57cec5SDimitry Andric void report(const char *msg, const MachineFunction *MF);
2450b57cec5SDimitry Andric void report(const char *msg, const MachineBasicBlock *MBB);
2460b57cec5SDimitry Andric void report(const char *msg, const MachineInstr *MI);
2470b57cec5SDimitry Andric void report(const char *msg, const MachineOperand *MO, unsigned MONum,
2480b57cec5SDimitry Andric LLT MOVRegType = LLT{});
249fe6060f1SDimitry Andric void report(const Twine &Msg, const MachineInstr *MI);
2500b57cec5SDimitry Andric
2510b57cec5SDimitry Andric void report_context(const LiveInterval &LI) const;
252e8d8bef9SDimitry Andric void report_context(const LiveRange &LR, Register VRegUnit,
2530b57cec5SDimitry Andric LaneBitmask LaneMask) const;
2540b57cec5SDimitry Andric void report_context(const LiveRange::Segment &S) const;
2550b57cec5SDimitry Andric void report_context(const VNInfo &VNI) const;
2560b57cec5SDimitry Andric void report_context(SlotIndex Pos) const;
2570b57cec5SDimitry Andric void report_context(MCPhysReg PhysReg) const;
2580b57cec5SDimitry Andric void report_context_liverange(const LiveRange &LR) const;
2590b57cec5SDimitry Andric void report_context_lanemask(LaneBitmask LaneMask) const;
260e8d8bef9SDimitry Andric void report_context_vreg(Register VReg) const;
261e8d8bef9SDimitry Andric void report_context_vreg_regunit(Register VRegOrUnit) const;
2620b57cec5SDimitry Andric
2630b57cec5SDimitry Andric void verifyInlineAsm(const MachineInstr *MI);
2640b57cec5SDimitry Andric
2650b57cec5SDimitry Andric void checkLiveness(const MachineOperand *MO, unsigned MONum);
2660b57cec5SDimitry Andric void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
267e8d8bef9SDimitry Andric SlotIndex UseIdx, const LiveRange &LR,
268e8d8bef9SDimitry Andric Register VRegOrUnit,
2690b57cec5SDimitry Andric LaneBitmask LaneMask = LaneBitmask::getNone());
2700b57cec5SDimitry Andric void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
271e8d8bef9SDimitry Andric SlotIndex DefIdx, const LiveRange &LR,
272e8d8bef9SDimitry Andric Register VRegOrUnit, bool SubRangeCheck = false,
2730b57cec5SDimitry Andric LaneBitmask LaneMask = LaneBitmask::getNone());
2740b57cec5SDimitry Andric
2750b57cec5SDimitry Andric void markReachable(const MachineBasicBlock *MBB);
2760b57cec5SDimitry Andric void calcRegsPassed();
2770b57cec5SDimitry Andric void checkPHIOps(const MachineBasicBlock &MBB);
2780b57cec5SDimitry Andric
2790b57cec5SDimitry Andric void calcRegsRequired();
2800b57cec5SDimitry Andric void verifyLiveVariables();
2810b57cec5SDimitry Andric void verifyLiveIntervals();
2820b57cec5SDimitry Andric void verifyLiveInterval(const LiveInterval&);
283e8d8bef9SDimitry Andric void verifyLiveRangeValue(const LiveRange &, const VNInfo *, Register,
2840b57cec5SDimitry Andric LaneBitmask);
2850b57cec5SDimitry Andric void verifyLiveRangeSegment(const LiveRange &,
286e8d8bef9SDimitry Andric const LiveRange::const_iterator I, Register,
2870b57cec5SDimitry Andric LaneBitmask);
288e8d8bef9SDimitry Andric void verifyLiveRange(const LiveRange &, Register,
2890b57cec5SDimitry Andric LaneBitmask LaneMask = LaneBitmask::getNone());
2900b57cec5SDimitry Andric
2910b57cec5SDimitry Andric void verifyStackFrame();
2920b57cec5SDimitry Andric
2930b57cec5SDimitry Andric void verifySlotIndexes() const;
2940b57cec5SDimitry Andric void verifyProperties(const MachineFunction &MF);
2950b57cec5SDimitry Andric };
2960b57cec5SDimitry Andric
2970b57cec5SDimitry Andric struct MachineVerifierPass : public MachineFunctionPass {
2980b57cec5SDimitry Andric static char ID; // Pass ID, replacement for typeid
2990b57cec5SDimitry Andric
3000b57cec5SDimitry Andric const std::string Banner;
3010b57cec5SDimitry Andric
MachineVerifierPass__anoncaf807cd0111::MachineVerifierPass3020b57cec5SDimitry Andric MachineVerifierPass(std::string banner = std::string())
3030b57cec5SDimitry Andric : MachineFunctionPass(ID), Banner(std::move(banner)) {
3040b57cec5SDimitry Andric initializeMachineVerifierPassPass(*PassRegistry::getPassRegistry());
3050b57cec5SDimitry Andric }
3060b57cec5SDimitry Andric
getAnalysisUsage__anoncaf807cd0111::MachineVerifierPass3070b57cec5SDimitry Andric void getAnalysisUsage(AnalysisUsage &AU) const override {
308753f127fSDimitry Andric AU.addUsedIfAvailable<LiveStacks>();
309bdd1243dSDimitry Andric AU.addUsedIfAvailable<LiveVariables>();
31006c3fb27SDimitry Andric AU.addUsedIfAvailable<SlotIndexes>();
31106c3fb27SDimitry Andric AU.addUsedIfAvailable<LiveIntervals>();
3120b57cec5SDimitry Andric AU.setPreservesAll();
3130b57cec5SDimitry Andric MachineFunctionPass::getAnalysisUsage(AU);
3140b57cec5SDimitry Andric }
3150b57cec5SDimitry Andric
runOnMachineFunction__anoncaf807cd0111::MachineVerifierPass3160b57cec5SDimitry Andric bool runOnMachineFunction(MachineFunction &MF) override {
317349cc55cSDimitry Andric // Skip functions that have known verification problems.
318349cc55cSDimitry Andric // FIXME: Remove this mechanism when all problematic passes have been
319349cc55cSDimitry Andric // fixed.
320349cc55cSDimitry Andric if (MF.getProperties().hasProperty(
321349cc55cSDimitry Andric MachineFunctionProperties::Property::FailsVerification))
322349cc55cSDimitry Andric return false;
323349cc55cSDimitry Andric
3240b57cec5SDimitry Andric unsigned FoundErrors = MachineVerifier(this, Banner.c_str()).verify(MF);
3250b57cec5SDimitry Andric if (FoundErrors)
3260b57cec5SDimitry Andric report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors.");
3270b57cec5SDimitry Andric return false;
3280b57cec5SDimitry Andric }
3290b57cec5SDimitry Andric };
3300b57cec5SDimitry Andric
3310b57cec5SDimitry Andric } // end anonymous namespace
3320b57cec5SDimitry Andric
3330b57cec5SDimitry Andric char MachineVerifierPass::ID = 0;
3340b57cec5SDimitry Andric
3350b57cec5SDimitry Andric INITIALIZE_PASS(MachineVerifierPass, "machineverifier",
3360b57cec5SDimitry Andric "Verify generated machine code", false, false)
3370b57cec5SDimitry Andric
createMachineVerifierPass(const std::string & Banner)3380b57cec5SDimitry Andric FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
3390b57cec5SDimitry Andric return new MachineVerifierPass(Banner);
3400b57cec5SDimitry Andric }
3410b57cec5SDimitry Andric
verifyMachineFunction(MachineFunctionAnalysisManager *,const std::string & Banner,const MachineFunction & MF)342e8d8bef9SDimitry Andric void llvm::verifyMachineFunction(MachineFunctionAnalysisManager *,
343e8d8bef9SDimitry Andric const std::string &Banner,
344e8d8bef9SDimitry Andric const MachineFunction &MF) {
345e8d8bef9SDimitry Andric // TODO: Use MFAM after porting below analyses.
346e8d8bef9SDimitry Andric // LiveVariables *LiveVars;
347e8d8bef9SDimitry Andric // LiveIntervals *LiveInts;
348e8d8bef9SDimitry Andric // LiveStacks *LiveStks;
349e8d8bef9SDimitry Andric // SlotIndexes *Indexes;
350e8d8bef9SDimitry Andric unsigned FoundErrors = MachineVerifier(nullptr, Banner.c_str()).verify(MF);
351e8d8bef9SDimitry Andric if (FoundErrors)
352e8d8bef9SDimitry Andric report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors.");
353e8d8bef9SDimitry Andric }
354e8d8bef9SDimitry Andric
verify(Pass * p,const char * Banner,bool AbortOnErrors) const3550b57cec5SDimitry Andric bool MachineFunction::verify(Pass *p, const char *Banner, bool AbortOnErrors)
3560b57cec5SDimitry Andric const {
3570b57cec5SDimitry Andric MachineFunction &MF = const_cast<MachineFunction&>(*this);
3580b57cec5SDimitry Andric unsigned FoundErrors = MachineVerifier(p, Banner).verify(MF);
3590b57cec5SDimitry Andric if (AbortOnErrors && FoundErrors)
3600b57cec5SDimitry Andric report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors.");
3610b57cec5SDimitry Andric return FoundErrors == 0;
3620b57cec5SDimitry Andric }
3630b57cec5SDimitry Andric
verify(LiveIntervals * LiveInts,SlotIndexes * Indexes,const char * Banner,bool AbortOnErrors) const3645f757f3fSDimitry Andric bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
3655f757f3fSDimitry Andric const char *Banner, bool AbortOnErrors) const {
3665f757f3fSDimitry Andric MachineFunction &MF = const_cast<MachineFunction &>(*this);
3675f757f3fSDimitry Andric unsigned FoundErrors =
3685f757f3fSDimitry Andric MachineVerifier(Banner, nullptr, LiveInts, nullptr, Indexes).verify(MF);
3695f757f3fSDimitry Andric if (AbortOnErrors && FoundErrors)
3705f757f3fSDimitry Andric report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors.");
3715f757f3fSDimitry Andric return FoundErrors == 0;
3725f757f3fSDimitry Andric }
3735f757f3fSDimitry Andric
verifySlotIndexes() const3740b57cec5SDimitry Andric void MachineVerifier::verifySlotIndexes() const {
3750b57cec5SDimitry Andric if (Indexes == nullptr)
3760b57cec5SDimitry Andric return;
3770b57cec5SDimitry Andric
3780b57cec5SDimitry Andric // Ensure the IdxMBB list is sorted by slot indexes.
3790b57cec5SDimitry Andric SlotIndex Last;
3800b57cec5SDimitry Andric for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
3810b57cec5SDimitry Andric E = Indexes->MBBIndexEnd(); I != E; ++I) {
3820b57cec5SDimitry Andric assert(!Last.isValid() || I->first > Last);
3830b57cec5SDimitry Andric Last = I->first;
3840b57cec5SDimitry Andric }
3850b57cec5SDimitry Andric }
3860b57cec5SDimitry Andric
verifyProperties(const MachineFunction & MF)3870b57cec5SDimitry Andric void MachineVerifier::verifyProperties(const MachineFunction &MF) {
3880b57cec5SDimitry Andric // If a pass has introduced virtual registers without clearing the
3890b57cec5SDimitry Andric // NoVRegs property (or set it without allocating the vregs)
3900b57cec5SDimitry Andric // then report an error.
3910b57cec5SDimitry Andric if (MF.getProperties().hasProperty(
3920b57cec5SDimitry Andric MachineFunctionProperties::Property::NoVRegs) &&
3930b57cec5SDimitry Andric MRI->getNumVirtRegs())
3940b57cec5SDimitry Andric report("Function has NoVRegs property but there are VReg operands", &MF);
3950b57cec5SDimitry Andric }
3960b57cec5SDimitry Andric
verify(const MachineFunction & MF)397e8d8bef9SDimitry Andric unsigned MachineVerifier::verify(const MachineFunction &MF) {
3980b57cec5SDimitry Andric foundErrors = 0;
3990b57cec5SDimitry Andric
4000b57cec5SDimitry Andric this->MF = &MF;
4010b57cec5SDimitry Andric TM = &MF.getTarget();
4020b57cec5SDimitry Andric TII = MF.getSubtarget().getInstrInfo();
4030b57cec5SDimitry Andric TRI = MF.getSubtarget().getRegisterInfo();
40481ad6265SDimitry Andric RBI = MF.getSubtarget().getRegBankInfo();
4050b57cec5SDimitry Andric MRI = &MF.getRegInfo();
4060b57cec5SDimitry Andric
4070b57cec5SDimitry Andric const bool isFunctionFailedISel = MF.getProperties().hasProperty(
4080b57cec5SDimitry Andric MachineFunctionProperties::Property::FailedISel);
4090b57cec5SDimitry Andric
4100b57cec5SDimitry Andric // If we're mid-GlobalISel and we already triggered the fallback path then
4110b57cec5SDimitry Andric // it's expected that the MIR is somewhat broken but that's ok since we'll
4120b57cec5SDimitry Andric // reset it and clear the FailedISel attribute in ResetMachineFunctions.
4130b57cec5SDimitry Andric if (isFunctionFailedISel)
4140b57cec5SDimitry Andric return foundErrors;
4150b57cec5SDimitry Andric
4165ffd83dbSDimitry Andric isFunctionRegBankSelected = MF.getProperties().hasProperty(
4170b57cec5SDimitry Andric MachineFunctionProperties::Property::RegBankSelected);
4185ffd83dbSDimitry Andric isFunctionSelected = MF.getProperties().hasProperty(
4190b57cec5SDimitry Andric MachineFunctionProperties::Property::Selected);
4200eae32dcSDimitry Andric isFunctionTracksDebugUserValues = MF.getProperties().hasProperty(
4210eae32dcSDimitry Andric MachineFunctionProperties::Property::TracksDebugUserValues);
4225ffd83dbSDimitry Andric
4230b57cec5SDimitry Andric if (PASS) {
4240b57cec5SDimitry Andric LiveInts = PASS->getAnalysisIfAvailable<LiveIntervals>();
4250b57cec5SDimitry Andric // We don't want to verify LiveVariables if LiveIntervals is available.
4260b57cec5SDimitry Andric if (!LiveInts)
4270b57cec5SDimitry Andric LiveVars = PASS->getAnalysisIfAvailable<LiveVariables>();
4280b57cec5SDimitry Andric LiveStks = PASS->getAnalysisIfAvailable<LiveStacks>();
4290b57cec5SDimitry Andric Indexes = PASS->getAnalysisIfAvailable<SlotIndexes>();
4300b57cec5SDimitry Andric }
4310b57cec5SDimitry Andric
4320b57cec5SDimitry Andric verifySlotIndexes();
4330b57cec5SDimitry Andric
4340b57cec5SDimitry Andric verifyProperties(MF);
4350b57cec5SDimitry Andric
4360b57cec5SDimitry Andric visitMachineFunctionBefore();
4375ffd83dbSDimitry Andric for (const MachineBasicBlock &MBB : MF) {
4385ffd83dbSDimitry Andric visitMachineBasicBlockBefore(&MBB);
4390b57cec5SDimitry Andric // Keep track of the current bundle header.
4400b57cec5SDimitry Andric const MachineInstr *CurBundle = nullptr;
4410b57cec5SDimitry Andric // Do we expect the next instruction to be part of the same bundle?
4420b57cec5SDimitry Andric bool InBundle = false;
4430b57cec5SDimitry Andric
4445ffd83dbSDimitry Andric for (const MachineInstr &MI : MBB.instrs()) {
4455ffd83dbSDimitry Andric if (MI.getParent() != &MBB) {
4465ffd83dbSDimitry Andric report("Bad instruction parent pointer", &MBB);
4475ffd83dbSDimitry Andric errs() << "Instruction: " << MI;
4480b57cec5SDimitry Andric continue;
4490b57cec5SDimitry Andric }
4500b57cec5SDimitry Andric
4510b57cec5SDimitry Andric // Check for consistent bundle flags.
4525ffd83dbSDimitry Andric if (InBundle && !MI.isBundledWithPred())
4530b57cec5SDimitry Andric report("Missing BundledPred flag, "
4540b57cec5SDimitry Andric "BundledSucc was set on predecessor",
4555ffd83dbSDimitry Andric &MI);
4565ffd83dbSDimitry Andric if (!InBundle && MI.isBundledWithPred())
4570b57cec5SDimitry Andric report("BundledPred flag is set, "
4580b57cec5SDimitry Andric "but BundledSucc not set on predecessor",
4595ffd83dbSDimitry Andric &MI);
4600b57cec5SDimitry Andric
4610b57cec5SDimitry Andric // Is this a bundle header?
4625ffd83dbSDimitry Andric if (!MI.isInsideBundle()) {
4630b57cec5SDimitry Andric if (CurBundle)
4640b57cec5SDimitry Andric visitMachineBundleAfter(CurBundle);
4655ffd83dbSDimitry Andric CurBundle = &MI;
4660b57cec5SDimitry Andric visitMachineBundleBefore(CurBundle);
4670b57cec5SDimitry Andric } else if (!CurBundle)
4685ffd83dbSDimitry Andric report("No bundle header", &MI);
4695ffd83dbSDimitry Andric visitMachineInstrBefore(&MI);
4705ffd83dbSDimitry Andric for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
4710b57cec5SDimitry Andric const MachineOperand &Op = MI.getOperand(I);
4720b57cec5SDimitry Andric if (Op.getParent() != &MI) {
47381ad6265SDimitry Andric // Make sure to use correct addOperand / removeOperand / ChangeTo
4740b57cec5SDimitry Andric // functions when replacing operands of a MachineInstr.
4750b57cec5SDimitry Andric report("Instruction has operand with wrong parent set", &MI);
4760b57cec5SDimitry Andric }
4770b57cec5SDimitry Andric
4780b57cec5SDimitry Andric visitMachineOperand(&Op, I);
4790b57cec5SDimitry Andric }
4800b57cec5SDimitry Andric
4810b57cec5SDimitry Andric // Was this the last bundled instruction?
4825ffd83dbSDimitry Andric InBundle = MI.isBundledWithSucc();
4830b57cec5SDimitry Andric }
4840b57cec5SDimitry Andric if (CurBundle)
4850b57cec5SDimitry Andric visitMachineBundleAfter(CurBundle);
4860b57cec5SDimitry Andric if (InBundle)
4875ffd83dbSDimitry Andric report("BundledSucc flag set on last instruction in block", &MBB.back());
4885ffd83dbSDimitry Andric visitMachineBasicBlockAfter(&MBB);
4890b57cec5SDimitry Andric }
4900b57cec5SDimitry Andric visitMachineFunctionAfter();
4910b57cec5SDimitry Andric
4920b57cec5SDimitry Andric // Clean up.
4930b57cec5SDimitry Andric regsLive.clear();
4940b57cec5SDimitry Andric regsDefined.clear();
4950b57cec5SDimitry Andric regsDead.clear();
4960b57cec5SDimitry Andric regsKilled.clear();
4970b57cec5SDimitry Andric regMasks.clear();
4980b57cec5SDimitry Andric MBBInfoMap.clear();
4990b57cec5SDimitry Andric
5000b57cec5SDimitry Andric return foundErrors;
5010b57cec5SDimitry Andric }
5020b57cec5SDimitry Andric
report(const char * msg,const MachineFunction * MF)5030b57cec5SDimitry Andric void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
5040b57cec5SDimitry Andric assert(MF);
5050b57cec5SDimitry Andric errs() << '\n';
5060b57cec5SDimitry Andric if (!foundErrors++) {
5070b57cec5SDimitry Andric if (Banner)
5080b57cec5SDimitry Andric errs() << "# " << Banner << '\n';
5090b57cec5SDimitry Andric if (LiveInts != nullptr)
5100b57cec5SDimitry Andric LiveInts->print(errs());
5110b57cec5SDimitry Andric else
5120b57cec5SDimitry Andric MF->print(errs(), Indexes);
5130b57cec5SDimitry Andric }
5140b57cec5SDimitry Andric errs() << "*** Bad machine code: " << msg << " ***\n"
5150b57cec5SDimitry Andric << "- function: " << MF->getName() << "\n";
5160b57cec5SDimitry Andric }
5170b57cec5SDimitry Andric
report(const char * msg,const MachineBasicBlock * MBB)5180b57cec5SDimitry Andric void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
5190b57cec5SDimitry Andric assert(MBB);
5200b57cec5SDimitry Andric report(msg, MBB->getParent());
5210b57cec5SDimitry Andric errs() << "- basic block: " << printMBBReference(*MBB) << ' '
5220b57cec5SDimitry Andric << MBB->getName() << " (" << (const void *)MBB << ')';
5230b57cec5SDimitry Andric if (Indexes)
5240b57cec5SDimitry Andric errs() << " [" << Indexes->getMBBStartIdx(MBB)
5250b57cec5SDimitry Andric << ';' << Indexes->getMBBEndIdx(MBB) << ')';
5260b57cec5SDimitry Andric errs() << '\n';
5270b57cec5SDimitry Andric }
5280b57cec5SDimitry Andric
report(const char * msg,const MachineInstr * MI)5290b57cec5SDimitry Andric void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
5300b57cec5SDimitry Andric assert(MI);
5310b57cec5SDimitry Andric report(msg, MI->getParent());
5320b57cec5SDimitry Andric errs() << "- instruction: ";
5330b57cec5SDimitry Andric if (Indexes && Indexes->hasIndex(*MI))
5340b57cec5SDimitry Andric errs() << Indexes->getInstructionIndex(*MI) << '\t';
535e8d8bef9SDimitry Andric MI->print(errs(), /*IsStandalone=*/true);
5360b57cec5SDimitry Andric }
5370b57cec5SDimitry Andric
report(const char * msg,const MachineOperand * MO,unsigned MONum,LLT MOVRegType)5380b57cec5SDimitry Andric void MachineVerifier::report(const char *msg, const MachineOperand *MO,
5390b57cec5SDimitry Andric unsigned MONum, LLT MOVRegType) {
5400b57cec5SDimitry Andric assert(MO);
5410b57cec5SDimitry Andric report(msg, MO->getParent());
5420b57cec5SDimitry Andric errs() << "- operand " << MONum << ": ";
5430b57cec5SDimitry Andric MO->print(errs(), MOVRegType, TRI);
5440b57cec5SDimitry Andric errs() << "\n";
5450b57cec5SDimitry Andric }
5460b57cec5SDimitry Andric
report(const Twine & Msg,const MachineInstr * MI)547fe6060f1SDimitry Andric void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
548fe6060f1SDimitry Andric report(Msg.str().c_str(), MI);
549fe6060f1SDimitry Andric }
550fe6060f1SDimitry Andric
report_context(SlotIndex Pos) const5510b57cec5SDimitry Andric void MachineVerifier::report_context(SlotIndex Pos) const {
5520b57cec5SDimitry Andric errs() << "- at: " << Pos << '\n';
5530b57cec5SDimitry Andric }
5540b57cec5SDimitry Andric
report_context(const LiveInterval & LI) const5550b57cec5SDimitry Andric void MachineVerifier::report_context(const LiveInterval &LI) const {
5560b57cec5SDimitry Andric errs() << "- interval: " << LI << '\n';
5570b57cec5SDimitry Andric }
5580b57cec5SDimitry Andric
report_context(const LiveRange & LR,Register VRegUnit,LaneBitmask LaneMask) const559e8d8bef9SDimitry Andric void MachineVerifier::report_context(const LiveRange &LR, Register VRegUnit,
5600b57cec5SDimitry Andric LaneBitmask LaneMask) const {
5610b57cec5SDimitry Andric report_context_liverange(LR);
5620b57cec5SDimitry Andric report_context_vreg_regunit(VRegUnit);
5630b57cec5SDimitry Andric if (LaneMask.any())
5640b57cec5SDimitry Andric report_context_lanemask(LaneMask);
5650b57cec5SDimitry Andric }
5660b57cec5SDimitry Andric
report_context(const LiveRange::Segment & S) const5670b57cec5SDimitry Andric void MachineVerifier::report_context(const LiveRange::Segment &S) const {
5680b57cec5SDimitry Andric errs() << "- segment: " << S << '\n';
5690b57cec5SDimitry Andric }
5700b57cec5SDimitry Andric
report_context(const VNInfo & VNI) const5710b57cec5SDimitry Andric void MachineVerifier::report_context(const VNInfo &VNI) const {
5720b57cec5SDimitry Andric errs() << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
5730b57cec5SDimitry Andric }
5740b57cec5SDimitry Andric
report_context_liverange(const LiveRange & LR) const5750b57cec5SDimitry Andric void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
5760b57cec5SDimitry Andric errs() << "- liverange: " << LR << '\n';
5770b57cec5SDimitry Andric }
5780b57cec5SDimitry Andric
report_context(MCPhysReg PReg) const5790b57cec5SDimitry Andric void MachineVerifier::report_context(MCPhysReg PReg) const {
5800b57cec5SDimitry Andric errs() << "- p. register: " << printReg(PReg, TRI) << '\n';
5810b57cec5SDimitry Andric }
5820b57cec5SDimitry Andric
report_context_vreg(Register VReg) const583e8d8bef9SDimitry Andric void MachineVerifier::report_context_vreg(Register VReg) const {
5840b57cec5SDimitry Andric errs() << "- v. register: " << printReg(VReg, TRI) << '\n';
5850b57cec5SDimitry Andric }
5860b57cec5SDimitry Andric
report_context_vreg_regunit(Register VRegOrUnit) const587e8d8bef9SDimitry Andric void MachineVerifier::report_context_vreg_regunit(Register VRegOrUnit) const {
588bdd1243dSDimitry Andric if (VRegOrUnit.isVirtual()) {
5890b57cec5SDimitry Andric report_context_vreg(VRegOrUnit);
5900b57cec5SDimitry Andric } else {
5910b57cec5SDimitry Andric errs() << "- regunit: " << printRegUnit(VRegOrUnit, TRI) << '\n';
5920b57cec5SDimitry Andric }
5930b57cec5SDimitry Andric }
5940b57cec5SDimitry Andric
report_context_lanemask(LaneBitmask LaneMask) const5950b57cec5SDimitry Andric void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
5960b57cec5SDimitry Andric errs() << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
5970b57cec5SDimitry Andric }
5980b57cec5SDimitry Andric
markReachable(const MachineBasicBlock * MBB)5990b57cec5SDimitry Andric void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
6000b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[MBB];
6010b57cec5SDimitry Andric if (!MInfo.reachable) {
6020b57cec5SDimitry Andric MInfo.reachable = true;
6035ffd83dbSDimitry Andric for (const MachineBasicBlock *Succ : MBB->successors())
6045ffd83dbSDimitry Andric markReachable(Succ);
6050b57cec5SDimitry Andric }
6060b57cec5SDimitry Andric }
6070b57cec5SDimitry Andric
visitMachineFunctionBefore()6080b57cec5SDimitry Andric void MachineVerifier::visitMachineFunctionBefore() {
6090b57cec5SDimitry Andric lastIndex = SlotIndex();
6100b57cec5SDimitry Andric regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
6110b57cec5SDimitry Andric : TRI->getReservedRegs(*MF);
6120b57cec5SDimitry Andric
6130b57cec5SDimitry Andric if (!MF->empty())
6140b57cec5SDimitry Andric markReachable(&MF->front());
6150b57cec5SDimitry Andric
6160b57cec5SDimitry Andric // Build a set of the basic blocks in the function.
6170b57cec5SDimitry Andric FunctionBlocks.clear();
6180b57cec5SDimitry Andric for (const auto &MBB : *MF) {
6190b57cec5SDimitry Andric FunctionBlocks.insert(&MBB);
6200b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MBB];
6210b57cec5SDimitry Andric
6220b57cec5SDimitry Andric MInfo.Preds.insert(MBB.pred_begin(), MBB.pred_end());
6230b57cec5SDimitry Andric if (MInfo.Preds.size() != MBB.pred_size())
6240b57cec5SDimitry Andric report("MBB has duplicate entries in its predecessor list.", &MBB);
6250b57cec5SDimitry Andric
6260b57cec5SDimitry Andric MInfo.Succs.insert(MBB.succ_begin(), MBB.succ_end());
6270b57cec5SDimitry Andric if (MInfo.Succs.size() != MBB.succ_size())
6280b57cec5SDimitry Andric report("MBB has duplicate entries in its successor list.", &MBB);
6290b57cec5SDimitry Andric }
6300b57cec5SDimitry Andric
6310b57cec5SDimitry Andric // Check that the register use lists are sane.
6320b57cec5SDimitry Andric MRI->verifyUseLists();
6330b57cec5SDimitry Andric
6340b57cec5SDimitry Andric if (!MF->empty())
6350b57cec5SDimitry Andric verifyStackFrame();
6360b57cec5SDimitry Andric }
6370b57cec5SDimitry Andric
6380b57cec5SDimitry Andric void
visitMachineBasicBlockBefore(const MachineBasicBlock * MBB)6390b57cec5SDimitry Andric MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
6400b57cec5SDimitry Andric FirstTerminator = nullptr;
6410b57cec5SDimitry Andric FirstNonPHI = nullptr;
6420b57cec5SDimitry Andric
6430b57cec5SDimitry Andric if (!MF->getProperties().hasProperty(
6440b57cec5SDimitry Andric MachineFunctionProperties::Property::NoPHIs) && MRI->tracksLiveness()) {
6450b57cec5SDimitry Andric // If this block has allocatable physical registers live-in, check that
6460b57cec5SDimitry Andric // it is an entry block or landing pad.
6470b57cec5SDimitry Andric for (const auto &LI : MBB->liveins()) {
6480b57cec5SDimitry Andric if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
64906c3fb27SDimitry Andric MBB->getIterator() != MBB->getParent()->begin() &&
65006c3fb27SDimitry Andric !MBB->isInlineAsmBrIndirectTarget()) {
65106c3fb27SDimitry Andric report("MBB has allocatable live-in, but isn't entry, landing-pad, or "
65206c3fb27SDimitry Andric "inlineasm-br-indirect-target.",
65306c3fb27SDimitry Andric MBB);
6540b57cec5SDimitry Andric report_context(LI.PhysReg);
6550b57cec5SDimitry Andric }
6560b57cec5SDimitry Andric }
6570b57cec5SDimitry Andric }
6580b57cec5SDimitry Andric
659bdd1243dSDimitry Andric if (MBB->isIRBlockAddressTaken()) {
660bdd1243dSDimitry Andric if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
661bdd1243dSDimitry Andric report("ir-block-address-taken is associated with basic block not used by "
662bdd1243dSDimitry Andric "a blockaddress.",
663bdd1243dSDimitry Andric MBB);
664bdd1243dSDimitry Andric }
665bdd1243dSDimitry Andric
6660b57cec5SDimitry Andric // Count the number of landing pad successors.
6675ffd83dbSDimitry Andric SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
6685ffd83dbSDimitry Andric for (const auto *succ : MBB->successors()) {
6695ffd83dbSDimitry Andric if (succ->isEHPad())
6705ffd83dbSDimitry Andric LandingPadSuccs.insert(succ);
6715ffd83dbSDimitry Andric if (!FunctionBlocks.count(succ))
6720b57cec5SDimitry Andric report("MBB has successor that isn't part of the function.", MBB);
6735ffd83dbSDimitry Andric if (!MBBInfoMap[succ].Preds.count(MBB)) {
6740b57cec5SDimitry Andric report("Inconsistent CFG", MBB);
6750b57cec5SDimitry Andric errs() << "MBB is not in the predecessor list of the successor "
6765ffd83dbSDimitry Andric << printMBBReference(*succ) << ".\n";
6770b57cec5SDimitry Andric }
6780b57cec5SDimitry Andric }
6790b57cec5SDimitry Andric
6800b57cec5SDimitry Andric // Check the predecessor list.
6815ffd83dbSDimitry Andric for (const MachineBasicBlock *Pred : MBB->predecessors()) {
6825ffd83dbSDimitry Andric if (!FunctionBlocks.count(Pred))
6830b57cec5SDimitry Andric report("MBB has predecessor that isn't part of the function.", MBB);
6845ffd83dbSDimitry Andric if (!MBBInfoMap[Pred].Succs.count(MBB)) {
6850b57cec5SDimitry Andric report("Inconsistent CFG", MBB);
6860b57cec5SDimitry Andric errs() << "MBB is not in the successor list of the predecessor "
6875ffd83dbSDimitry Andric << printMBBReference(*Pred) << ".\n";
6880b57cec5SDimitry Andric }
6890b57cec5SDimitry Andric }
6900b57cec5SDimitry Andric
6910b57cec5SDimitry Andric const MCAsmInfo *AsmInfo = TM->getMCAsmInfo();
6920b57cec5SDimitry Andric const BasicBlock *BB = MBB->getBasicBlock();
6930b57cec5SDimitry Andric const Function &F = MF->getFunction();
6940b57cec5SDimitry Andric if (LandingPadSuccs.size() > 1 &&
6950b57cec5SDimitry Andric !(AsmInfo &&
6960b57cec5SDimitry Andric AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj &&
6970b57cec5SDimitry Andric BB && isa<SwitchInst>(BB->getTerminator())) &&
6980b57cec5SDimitry Andric !isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn())))
6990b57cec5SDimitry Andric report("MBB has more than one landing pad successor", MBB);
7000b57cec5SDimitry Andric
7015ffd83dbSDimitry Andric // Call analyzeBranch. If it succeeds, there several more conditions to check.
7020b57cec5SDimitry Andric MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
7030b57cec5SDimitry Andric SmallVector<MachineOperand, 4> Cond;
7040b57cec5SDimitry Andric if (!TII->analyzeBranch(*const_cast<MachineBasicBlock *>(MBB), TBB, FBB,
7050b57cec5SDimitry Andric Cond)) {
7065ffd83dbSDimitry Andric // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
7070b57cec5SDimitry Andric // check whether its answers match up with reality.
7080b57cec5SDimitry Andric if (!TBB && !FBB) {
7090b57cec5SDimitry Andric // Block falls through to its successor.
7100b57cec5SDimitry Andric if (!MBB->empty() && MBB->back().isBarrier() &&
7110b57cec5SDimitry Andric !TII->isPredicated(MBB->back())) {
7120b57cec5SDimitry Andric report("MBB exits via unconditional fall-through but ends with a "
7130b57cec5SDimitry Andric "barrier instruction!", MBB);
7140b57cec5SDimitry Andric }
7150b57cec5SDimitry Andric if (!Cond.empty()) {
7160b57cec5SDimitry Andric report("MBB exits via unconditional fall-through but has a condition!",
7170b57cec5SDimitry Andric MBB);
7180b57cec5SDimitry Andric }
7190b57cec5SDimitry Andric } else if (TBB && !FBB && Cond.empty()) {
7200b57cec5SDimitry Andric // Block unconditionally branches somewhere.
7210b57cec5SDimitry Andric if (MBB->empty()) {
7220b57cec5SDimitry Andric report("MBB exits via unconditional branch but doesn't contain "
7230b57cec5SDimitry Andric "any instructions!", MBB);
7240b57cec5SDimitry Andric } else if (!MBB->back().isBarrier()) {
7250b57cec5SDimitry Andric report("MBB exits via unconditional branch but doesn't end with a "
7260b57cec5SDimitry Andric "barrier instruction!", MBB);
7270b57cec5SDimitry Andric } else if (!MBB->back().isTerminator()) {
7280b57cec5SDimitry Andric report("MBB exits via unconditional branch but the branch isn't a "
7290b57cec5SDimitry Andric "terminator instruction!", MBB);
7300b57cec5SDimitry Andric }
7310b57cec5SDimitry Andric } else if (TBB && !FBB && !Cond.empty()) {
7320b57cec5SDimitry Andric // Block conditionally branches somewhere, otherwise falls through.
7330b57cec5SDimitry Andric if (MBB->empty()) {
7340b57cec5SDimitry Andric report("MBB exits via conditional branch/fall-through but doesn't "
7350b57cec5SDimitry Andric "contain any instructions!", MBB);
7360b57cec5SDimitry Andric } else if (MBB->back().isBarrier()) {
7370b57cec5SDimitry Andric report("MBB exits via conditional branch/fall-through but ends with a "
7380b57cec5SDimitry Andric "barrier instruction!", MBB);
7390b57cec5SDimitry Andric } else if (!MBB->back().isTerminator()) {
7400b57cec5SDimitry Andric report("MBB exits via conditional branch/fall-through but the branch "
7410b57cec5SDimitry Andric "isn't a terminator instruction!", MBB);
7420b57cec5SDimitry Andric }
7430b57cec5SDimitry Andric } else if (TBB && FBB) {
7440b57cec5SDimitry Andric // Block conditionally branches somewhere, otherwise branches
7450b57cec5SDimitry Andric // somewhere else.
7460b57cec5SDimitry Andric if (MBB->empty()) {
7470b57cec5SDimitry Andric report("MBB exits via conditional branch/branch but doesn't "
7480b57cec5SDimitry Andric "contain any instructions!", MBB);
7490b57cec5SDimitry Andric } else if (!MBB->back().isBarrier()) {
7500b57cec5SDimitry Andric report("MBB exits via conditional branch/branch but doesn't end with a "
7510b57cec5SDimitry Andric "barrier instruction!", MBB);
7520b57cec5SDimitry Andric } else if (!MBB->back().isTerminator()) {
7530b57cec5SDimitry Andric report("MBB exits via conditional branch/branch but the branch "
7540b57cec5SDimitry Andric "isn't a terminator instruction!", MBB);
7550b57cec5SDimitry Andric }
7560b57cec5SDimitry Andric if (Cond.empty()) {
7570b57cec5SDimitry Andric report("MBB exits via conditional branch/branch but there's no "
7580b57cec5SDimitry Andric "condition!", MBB);
7590b57cec5SDimitry Andric }
7600b57cec5SDimitry Andric } else {
7615ffd83dbSDimitry Andric report("analyzeBranch returned invalid data!", MBB);
7625ffd83dbSDimitry Andric }
7635ffd83dbSDimitry Andric
7645ffd83dbSDimitry Andric // Now check that the successors match up with the answers reported by
7655ffd83dbSDimitry Andric // analyzeBranch.
7665ffd83dbSDimitry Andric if (TBB && !MBB->isSuccessor(TBB))
7675ffd83dbSDimitry Andric report("MBB exits via jump or conditional branch, but its target isn't a "
7685ffd83dbSDimitry Andric "CFG successor!",
7695ffd83dbSDimitry Andric MBB);
7705ffd83dbSDimitry Andric if (FBB && !MBB->isSuccessor(FBB))
7715ffd83dbSDimitry Andric report("MBB exits via conditional branch, but its target isn't a CFG "
7725ffd83dbSDimitry Andric "successor!",
7735ffd83dbSDimitry Andric MBB);
7745ffd83dbSDimitry Andric
7755ffd83dbSDimitry Andric // There might be a fallthrough to the next block if there's either no
7765ffd83dbSDimitry Andric // unconditional true branch, or if there's a condition, and one of the
7775ffd83dbSDimitry Andric // branches is missing.
7785ffd83dbSDimitry Andric bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
7795ffd83dbSDimitry Andric
7805ffd83dbSDimitry Andric // A conditional fallthrough must be an actual CFG successor, not
7815ffd83dbSDimitry Andric // unreachable. (Conversely, an unconditional fallthrough might not really
7825ffd83dbSDimitry Andric // be a successor, because the block might end in unreachable.)
7835ffd83dbSDimitry Andric if (!Cond.empty() && !FBB) {
7845ffd83dbSDimitry Andric MachineFunction::const_iterator MBBI = std::next(MBB->getIterator());
7855ffd83dbSDimitry Andric if (MBBI == MF->end()) {
7865ffd83dbSDimitry Andric report("MBB conditionally falls through out of function!", MBB);
7875ffd83dbSDimitry Andric } else if (!MBB->isSuccessor(&*MBBI))
7885ffd83dbSDimitry Andric report("MBB exits via conditional branch/fall-through but the CFG "
7895ffd83dbSDimitry Andric "successors don't match the actual successors!",
7905ffd83dbSDimitry Andric MBB);
7915ffd83dbSDimitry Andric }
7925ffd83dbSDimitry Andric
7935ffd83dbSDimitry Andric // Verify that there aren't any extra un-accounted-for successors.
7945ffd83dbSDimitry Andric for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
7955ffd83dbSDimitry Andric // If this successor is one of the branch targets, it's okay.
7965ffd83dbSDimitry Andric if (SuccMBB == TBB || SuccMBB == FBB)
7975ffd83dbSDimitry Andric continue;
7985ffd83dbSDimitry Andric // If we might have a fallthrough, and the successor is the fallthrough
7995ffd83dbSDimitry Andric // block, that's also ok.
8005ffd83dbSDimitry Andric if (Fallthrough && SuccMBB == MBB->getNextNode())
8015ffd83dbSDimitry Andric continue;
8025ffd83dbSDimitry Andric // Also accept successors which are for exception-handling or might be
8035ffd83dbSDimitry Andric // inlineasm_br targets.
8045ffd83dbSDimitry Andric if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
8055ffd83dbSDimitry Andric continue;
8065ffd83dbSDimitry Andric report("MBB has unexpected successors which are not branch targets, "
8075ffd83dbSDimitry Andric "fallthrough, EHPads, or inlineasm_br targets.",
8085ffd83dbSDimitry Andric MBB);
8090b57cec5SDimitry Andric }
8100b57cec5SDimitry Andric }
8110b57cec5SDimitry Andric
8120b57cec5SDimitry Andric regsLive.clear();
8130b57cec5SDimitry Andric if (MRI->tracksLiveness()) {
8140b57cec5SDimitry Andric for (const auto &LI : MBB->liveins()) {
8158bcb0991SDimitry Andric if (!Register::isPhysicalRegister(LI.PhysReg)) {
8160b57cec5SDimitry Andric report("MBB live-in list contains non-physical register", MBB);
8170b57cec5SDimitry Andric continue;
8180b57cec5SDimitry Andric }
819480093f4SDimitry Andric for (const MCPhysReg &SubReg : TRI->subregs_inclusive(LI.PhysReg))
820480093f4SDimitry Andric regsLive.insert(SubReg);
8210b57cec5SDimitry Andric }
8220b57cec5SDimitry Andric }
8230b57cec5SDimitry Andric
8240b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF->getFrameInfo();
8250b57cec5SDimitry Andric BitVector PR = MFI.getPristineRegs(*MF);
8260b57cec5SDimitry Andric for (unsigned I : PR.set_bits()) {
827480093f4SDimitry Andric for (const MCPhysReg &SubReg : TRI->subregs_inclusive(I))
828480093f4SDimitry Andric regsLive.insert(SubReg);
8290b57cec5SDimitry Andric }
8300b57cec5SDimitry Andric
8310b57cec5SDimitry Andric regsKilled.clear();
8320b57cec5SDimitry Andric regsDefined.clear();
8330b57cec5SDimitry Andric
8340b57cec5SDimitry Andric if (Indexes)
8350b57cec5SDimitry Andric lastIndex = Indexes->getMBBStartIdx(MBB);
8360b57cec5SDimitry Andric }
8370b57cec5SDimitry Andric
8380b57cec5SDimitry Andric // This function gets called for all bundle headers, including normal
8390b57cec5SDimitry Andric // stand-alone unbundled instructions.
visitMachineBundleBefore(const MachineInstr * MI)8400b57cec5SDimitry Andric void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
8410b57cec5SDimitry Andric if (Indexes && Indexes->hasIndex(*MI)) {
8420b57cec5SDimitry Andric SlotIndex idx = Indexes->getInstructionIndex(*MI);
8430b57cec5SDimitry Andric if (!(idx > lastIndex)) {
8440b57cec5SDimitry Andric report("Instruction index out of order", MI);
8450b57cec5SDimitry Andric errs() << "Last instruction was at " << lastIndex << '\n';
8460b57cec5SDimitry Andric }
8470b57cec5SDimitry Andric lastIndex = idx;
8480b57cec5SDimitry Andric }
8490b57cec5SDimitry Andric
8500b57cec5SDimitry Andric // Ensure non-terminators don't follow terminators.
851e8d8bef9SDimitry Andric if (MI->isTerminator()) {
8520b57cec5SDimitry Andric if (!FirstTerminator)
8530b57cec5SDimitry Andric FirstTerminator = MI;
8545ffd83dbSDimitry Andric } else if (FirstTerminator) {
855bdd1243dSDimitry Andric // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
856bdd1243dSDimitry Andric // precede non-terminators.
857bdd1243dSDimitry Andric if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
8580b57cec5SDimitry Andric report("Non-terminator instruction after the first terminator", MI);
8590b57cec5SDimitry Andric errs() << "First terminator was:\t" << *FirstTerminator;
8600b57cec5SDimitry Andric }
8610b57cec5SDimitry Andric }
862bdd1243dSDimitry Andric }
8630b57cec5SDimitry Andric
8640b57cec5SDimitry Andric // The operands on an INLINEASM instruction must follow a template.
8650b57cec5SDimitry Andric // Verify that the flag operands make sense.
verifyInlineAsm(const MachineInstr * MI)8660b57cec5SDimitry Andric void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
8670b57cec5SDimitry Andric // The first two operands on INLINEASM are the asm string and global flags.
8680b57cec5SDimitry Andric if (MI->getNumOperands() < 2) {
8690b57cec5SDimitry Andric report("Too few operands on inline asm", MI);
8700b57cec5SDimitry Andric return;
8710b57cec5SDimitry Andric }
8720b57cec5SDimitry Andric if (!MI->getOperand(0).isSymbol())
8730b57cec5SDimitry Andric report("Asm string must be an external symbol", MI);
8740b57cec5SDimitry Andric if (!MI->getOperand(1).isImm())
8750b57cec5SDimitry Andric report("Asm flags must be an immediate", MI);
8760b57cec5SDimitry Andric // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
8770b57cec5SDimitry Andric // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
8780b57cec5SDimitry Andric // and Extra_IsConvergent = 32.
8790b57cec5SDimitry Andric if (!isUInt<6>(MI->getOperand(1).getImm()))
8800b57cec5SDimitry Andric report("Unknown asm flags", &MI->getOperand(1), 1);
8810b57cec5SDimitry Andric
8820b57cec5SDimitry Andric static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
8830b57cec5SDimitry Andric
8840b57cec5SDimitry Andric unsigned OpNo = InlineAsm::MIOp_FirstOperand;
8850b57cec5SDimitry Andric unsigned NumOps;
8860b57cec5SDimitry Andric for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
8870b57cec5SDimitry Andric const MachineOperand &MO = MI->getOperand(OpNo);
8880b57cec5SDimitry Andric // There may be implicit ops after the fixed operands.
8890b57cec5SDimitry Andric if (!MO.isImm())
8900b57cec5SDimitry Andric break;
8915f757f3fSDimitry Andric const InlineAsm::Flag F(MO.getImm());
8925f757f3fSDimitry Andric NumOps = 1 + F.getNumOperandRegisters();
8930b57cec5SDimitry Andric }
8940b57cec5SDimitry Andric
8950b57cec5SDimitry Andric if (OpNo > MI->getNumOperands())
8960b57cec5SDimitry Andric report("Missing operands in last group", MI);
8970b57cec5SDimitry Andric
8980b57cec5SDimitry Andric // An optional MDNode follows the groups.
8990b57cec5SDimitry Andric if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata())
9000b57cec5SDimitry Andric ++OpNo;
9010b57cec5SDimitry Andric
9020b57cec5SDimitry Andric // All trailing operands must be implicit registers.
9030b57cec5SDimitry Andric for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
9040b57cec5SDimitry Andric const MachineOperand &MO = MI->getOperand(OpNo);
9050b57cec5SDimitry Andric if (!MO.isReg() || !MO.isImplicit())
9060b57cec5SDimitry Andric report("Expected implicit register after groups", &MO, OpNo);
9070b57cec5SDimitry Andric }
908bdd1243dSDimitry Andric
909bdd1243dSDimitry Andric if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
910bdd1243dSDimitry Andric const MachineBasicBlock *MBB = MI->getParent();
911bdd1243dSDimitry Andric
912bdd1243dSDimitry Andric for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
913bdd1243dSDimitry Andric i != e; ++i) {
914bdd1243dSDimitry Andric const MachineOperand &MO = MI->getOperand(i);
915bdd1243dSDimitry Andric
916bdd1243dSDimitry Andric if (!MO.isMBB())
917bdd1243dSDimitry Andric continue;
918bdd1243dSDimitry Andric
919bdd1243dSDimitry Andric // Check the successor & predecessor lists look ok, assume they are
920bdd1243dSDimitry Andric // not. Find the indirect target without going through the successors.
921bdd1243dSDimitry Andric const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
922bdd1243dSDimitry Andric if (!IndirectTargetMBB) {
923bdd1243dSDimitry Andric report("INLINEASM_BR indirect target does not exist", &MO, i);
924bdd1243dSDimitry Andric break;
925bdd1243dSDimitry Andric }
926bdd1243dSDimitry Andric
927bdd1243dSDimitry Andric if (!MBB->isSuccessor(IndirectTargetMBB))
928bdd1243dSDimitry Andric report("INLINEASM_BR indirect target missing from successor list", &MO,
929bdd1243dSDimitry Andric i);
930bdd1243dSDimitry Andric
931bdd1243dSDimitry Andric if (!IndirectTargetMBB->isPredecessor(MBB))
932bdd1243dSDimitry Andric report("INLINEASM_BR indirect target predecessor list missing parent",
933bdd1243dSDimitry Andric &MO, i);
934bdd1243dSDimitry Andric }
935bdd1243dSDimitry Andric }
9360b57cec5SDimitry Andric }
9370b57cec5SDimitry Andric
verifyAllRegOpsScalar(const MachineInstr & MI,const MachineRegisterInfo & MRI)938349cc55cSDimitry Andric bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
939349cc55cSDimitry Andric const MachineRegisterInfo &MRI) {
940349cc55cSDimitry Andric if (none_of(MI.explicit_operands(), [&MRI](const MachineOperand &Op) {
941349cc55cSDimitry Andric if (!Op.isReg())
942349cc55cSDimitry Andric return false;
943349cc55cSDimitry Andric const auto Reg = Op.getReg();
944349cc55cSDimitry Andric if (Reg.isPhysical())
945349cc55cSDimitry Andric return false;
946349cc55cSDimitry Andric return !MRI.getType(Reg).isScalar();
947349cc55cSDimitry Andric }))
948349cc55cSDimitry Andric return true;
949349cc55cSDimitry Andric report("All register operands must have scalar types", &MI);
950349cc55cSDimitry Andric return false;
951349cc55cSDimitry Andric }
952349cc55cSDimitry Andric
9530b57cec5SDimitry Andric /// Check that types are consistent when two operands need to have the same
9540b57cec5SDimitry Andric /// number of vector elements.
9550b57cec5SDimitry Andric /// \return true if the types are valid.
verifyVectorElementMatch(LLT Ty0,LLT Ty1,const MachineInstr * MI)9560b57cec5SDimitry Andric bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
9570b57cec5SDimitry Andric const MachineInstr *MI) {
9580b57cec5SDimitry Andric if (Ty0.isVector() != Ty1.isVector()) {
9590b57cec5SDimitry Andric report("operand types must be all-vector or all-scalar", MI);
9600b57cec5SDimitry Andric // Generally we try to report as many issues as possible at once, but in
9610b57cec5SDimitry Andric // this case it's not clear what should we be comparing the size of the
9620b57cec5SDimitry Andric // scalar with: the size of the whole vector or its lane. Instead of
9630b57cec5SDimitry Andric // making an arbitrary choice and emitting not so helpful message, let's
9640b57cec5SDimitry Andric // avoid the extra noise and stop here.
9650b57cec5SDimitry Andric return false;
9660b57cec5SDimitry Andric }
9670b57cec5SDimitry Andric
9685f757f3fSDimitry Andric if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
9690b57cec5SDimitry Andric report("operand types must preserve number of vector elements", MI);
9700b57cec5SDimitry Andric return false;
9710b57cec5SDimitry Andric }
9720b57cec5SDimitry Andric
9730b57cec5SDimitry Andric return true;
9740b57cec5SDimitry Andric }
9750b57cec5SDimitry Andric
verifyGIntrinsicSideEffects(const MachineInstr * MI)9765f757f3fSDimitry Andric bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
9775f757f3fSDimitry Andric auto Opcode = MI->getOpcode();
9785f757f3fSDimitry Andric bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
9795f757f3fSDimitry Andric Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
9805f757f3fSDimitry Andric unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
9815f757f3fSDimitry Andric if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
9825f757f3fSDimitry Andric AttributeList Attrs = Intrinsic::getAttributes(
9835f757f3fSDimitry Andric MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
9845f757f3fSDimitry Andric bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
9855f757f3fSDimitry Andric if (NoSideEffects && DeclHasSideEffects) {
9865f757f3fSDimitry Andric report(Twine(TII->getName(Opcode),
9875f757f3fSDimitry Andric " used with intrinsic that accesses memory"),
9885f757f3fSDimitry Andric MI);
9895f757f3fSDimitry Andric return false;
9905f757f3fSDimitry Andric }
9915f757f3fSDimitry Andric if (!NoSideEffects && !DeclHasSideEffects) {
9925f757f3fSDimitry Andric report(Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
9935f757f3fSDimitry Andric return false;
9945f757f3fSDimitry Andric }
9955f757f3fSDimitry Andric }
9965f757f3fSDimitry Andric
9975f757f3fSDimitry Andric return true;
9985f757f3fSDimitry Andric }
9995f757f3fSDimitry Andric
verifyGIntrinsicConvergence(const MachineInstr * MI)10005f757f3fSDimitry Andric bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
10015f757f3fSDimitry Andric auto Opcode = MI->getOpcode();
10025f757f3fSDimitry Andric bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
10035f757f3fSDimitry Andric Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
10045f757f3fSDimitry Andric unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
10055f757f3fSDimitry Andric if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
10065f757f3fSDimitry Andric AttributeList Attrs = Intrinsic::getAttributes(
10075f757f3fSDimitry Andric MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
10085f757f3fSDimitry Andric bool DeclIsConvergent = Attrs.hasFnAttr(Attribute::Convergent);
10095f757f3fSDimitry Andric if (NotConvergent && DeclIsConvergent) {
10105f757f3fSDimitry Andric report(Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
10115f757f3fSDimitry Andric MI);
10125f757f3fSDimitry Andric return false;
10135f757f3fSDimitry Andric }
10145f757f3fSDimitry Andric if (!NotConvergent && !DeclIsConvergent) {
10155f757f3fSDimitry Andric report(
10165f757f3fSDimitry Andric Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
10175f757f3fSDimitry Andric MI);
10185f757f3fSDimitry Andric return false;
10195f757f3fSDimitry Andric }
10205f757f3fSDimitry Andric }
10215f757f3fSDimitry Andric
10225f757f3fSDimitry Andric return true;
10235f757f3fSDimitry Andric }
10245f757f3fSDimitry Andric
verifyPreISelGenericInstruction(const MachineInstr * MI)10250b57cec5SDimitry Andric void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
10260b57cec5SDimitry Andric if (isFunctionSelected)
10270b57cec5SDimitry Andric report("Unexpected generic instruction in a Selected function", MI);
10280b57cec5SDimitry Andric
10290b57cec5SDimitry Andric const MCInstrDesc &MCID = MI->getDesc();
10300b57cec5SDimitry Andric unsigned NumOps = MI->getNumOperands();
10310b57cec5SDimitry Andric
10325ffd83dbSDimitry Andric // Branches must reference a basic block if they are not indirect
10335ffd83dbSDimitry Andric if (MI->isBranch() && !MI->isIndirectBranch()) {
10345ffd83dbSDimitry Andric bool HasMBB = false;
10355ffd83dbSDimitry Andric for (const MachineOperand &Op : MI->operands()) {
10365ffd83dbSDimitry Andric if (Op.isMBB()) {
10375ffd83dbSDimitry Andric HasMBB = true;
10385ffd83dbSDimitry Andric break;
10395ffd83dbSDimitry Andric }
10405ffd83dbSDimitry Andric }
10415ffd83dbSDimitry Andric
10425ffd83dbSDimitry Andric if (!HasMBB) {
10435ffd83dbSDimitry Andric report("Branch instruction is missing a basic block operand or "
10445ffd83dbSDimitry Andric "isIndirectBranch property",
10455ffd83dbSDimitry Andric MI);
10465ffd83dbSDimitry Andric }
10475ffd83dbSDimitry Andric }
10485ffd83dbSDimitry Andric
10490b57cec5SDimitry Andric // Check types.
10500b57cec5SDimitry Andric SmallVector<LLT, 4> Types;
10510b57cec5SDimitry Andric for (unsigned I = 0, E = std::min(MCID.getNumOperands(), NumOps);
10520b57cec5SDimitry Andric I != E; ++I) {
1053bdd1243dSDimitry Andric if (!MCID.operands()[I].isGenericType())
10540b57cec5SDimitry Andric continue;
10550b57cec5SDimitry Andric // Generic instructions specify type equality constraints between some of
10560b57cec5SDimitry Andric // their operands. Make sure these are consistent.
1057bdd1243dSDimitry Andric size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
10580b57cec5SDimitry Andric Types.resize(std::max(TypeIdx + 1, Types.size()));
10590b57cec5SDimitry Andric
10600b57cec5SDimitry Andric const MachineOperand *MO = &MI->getOperand(I);
10610b57cec5SDimitry Andric if (!MO->isReg()) {
10620b57cec5SDimitry Andric report("generic instruction must use register operands", MI);
10630b57cec5SDimitry Andric continue;
10640b57cec5SDimitry Andric }
10650b57cec5SDimitry Andric
10660b57cec5SDimitry Andric LLT OpTy = MRI->getType(MO->getReg());
10670b57cec5SDimitry Andric // Don't report a type mismatch if there is no actual mismatch, only a
10680b57cec5SDimitry Andric // type missing, to reduce noise:
10690b57cec5SDimitry Andric if (OpTy.isValid()) {
10700b57cec5SDimitry Andric // Only the first valid type for a type index will be printed: don't
10710b57cec5SDimitry Andric // overwrite it later so it's always clear which type was expected:
10720b57cec5SDimitry Andric if (!Types[TypeIdx].isValid())
10730b57cec5SDimitry Andric Types[TypeIdx] = OpTy;
10740b57cec5SDimitry Andric else if (Types[TypeIdx] != OpTy)
10750b57cec5SDimitry Andric report("Type mismatch in generic instruction", MO, I, OpTy);
10760b57cec5SDimitry Andric } else {
10770b57cec5SDimitry Andric // Generic instructions must have types attached to their operands.
10780b57cec5SDimitry Andric report("Generic instruction is missing a virtual register type", MO, I);
10790b57cec5SDimitry Andric }
10800b57cec5SDimitry Andric }
10810b57cec5SDimitry Andric
10820b57cec5SDimitry Andric // Generic opcodes must not have physical register operands.
10830b57cec5SDimitry Andric for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
10840b57cec5SDimitry Andric const MachineOperand *MO = &MI->getOperand(I);
1085bdd1243dSDimitry Andric if (MO->isReg() && MO->getReg().isPhysical())
10860b57cec5SDimitry Andric report("Generic instruction cannot have physical register", MO, I);
10870b57cec5SDimitry Andric }
10880b57cec5SDimitry Andric
10890b57cec5SDimitry Andric // Avoid out of bounds in checks below. This was already reported earlier.
10900b57cec5SDimitry Andric if (MI->getNumOperands() < MCID.getNumOperands())
10910b57cec5SDimitry Andric return;
10920b57cec5SDimitry Andric
10930b57cec5SDimitry Andric StringRef ErrorInfo;
10940b57cec5SDimitry Andric if (!TII->verifyInstruction(*MI, ErrorInfo))
10950b57cec5SDimitry Andric report(ErrorInfo.data(), MI);
10960b57cec5SDimitry Andric
10970b57cec5SDimitry Andric // Verify properties of various specific instruction types
1098fe6060f1SDimitry Andric unsigned Opc = MI->getOpcode();
1099fe6060f1SDimitry Andric switch (Opc) {
1100fe6060f1SDimitry Andric case TargetOpcode::G_ASSERT_SEXT:
1101fe6060f1SDimitry Andric case TargetOpcode::G_ASSERT_ZEXT: {
1102fe6060f1SDimitry Andric std::string OpcName =
1103fe6060f1SDimitry Andric Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1104fe6060f1SDimitry Andric if (!MI->getOperand(2).isImm()) {
1105fe6060f1SDimitry Andric report(Twine(OpcName, " expects an immediate operand #2"), MI);
1106fe6060f1SDimitry Andric break;
1107fe6060f1SDimitry Andric }
1108fe6060f1SDimitry Andric
1109fe6060f1SDimitry Andric Register Dst = MI->getOperand(0).getReg();
1110fe6060f1SDimitry Andric Register Src = MI->getOperand(1).getReg();
1111fe6060f1SDimitry Andric LLT SrcTy = MRI->getType(Src);
1112fe6060f1SDimitry Andric int64_t Imm = MI->getOperand(2).getImm();
1113fe6060f1SDimitry Andric if (Imm <= 0) {
1114fe6060f1SDimitry Andric report(Twine(OpcName, " size must be >= 1"), MI);
1115fe6060f1SDimitry Andric break;
1116fe6060f1SDimitry Andric }
1117fe6060f1SDimitry Andric
1118fe6060f1SDimitry Andric if (Imm >= SrcTy.getScalarSizeInBits()) {
1119fe6060f1SDimitry Andric report(Twine(OpcName, " size must be less than source bit width"), MI);
1120fe6060f1SDimitry Andric break;
1121fe6060f1SDimitry Andric }
1122fe6060f1SDimitry Andric
112381ad6265SDimitry Andric const RegisterBank *SrcRB = RBI->getRegBank(Src, *MRI, *TRI);
112481ad6265SDimitry Andric const RegisterBank *DstRB = RBI->getRegBank(Dst, *MRI, *TRI);
112581ad6265SDimitry Andric
112681ad6265SDimitry Andric // Allow only the source bank to be set.
112781ad6265SDimitry Andric if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
112881ad6265SDimitry Andric report(Twine(OpcName, " cannot change register bank"), MI);
1129fe6060f1SDimitry Andric break;
1130fe6060f1SDimitry Andric }
1131fe6060f1SDimitry Andric
113281ad6265SDimitry Andric // Don't allow a class change. Do allow member class->regbank.
113381ad6265SDimitry Andric const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Dst);
113481ad6265SDimitry Andric if (DstRC && DstRC != MRI->getRegClassOrNull(Src)) {
1135fe6060f1SDimitry Andric report(
1136fe6060f1SDimitry Andric Twine(OpcName, " source and destination register classes must match"),
1137fe6060f1SDimitry Andric MI);
113881ad6265SDimitry Andric break;
113981ad6265SDimitry Andric }
1140fe6060f1SDimitry Andric
1141fe6060f1SDimitry Andric break;
1142fe6060f1SDimitry Andric }
1143fe6060f1SDimitry Andric
11440b57cec5SDimitry Andric case TargetOpcode::G_CONSTANT:
11450b57cec5SDimitry Andric case TargetOpcode::G_FCONSTANT: {
11460b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
11470b57cec5SDimitry Andric if (DstTy.isVector())
11480b57cec5SDimitry Andric report("Instruction cannot use a vector result type", MI);
11490b57cec5SDimitry Andric
11500b57cec5SDimitry Andric if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
11510b57cec5SDimitry Andric if (!MI->getOperand(1).isCImm()) {
11520b57cec5SDimitry Andric report("G_CONSTANT operand must be cimm", MI);
11530b57cec5SDimitry Andric break;
11540b57cec5SDimitry Andric }
11550b57cec5SDimitry Andric
11560b57cec5SDimitry Andric const ConstantInt *CI = MI->getOperand(1).getCImm();
11570b57cec5SDimitry Andric if (CI->getBitWidth() != DstTy.getSizeInBits())
11580b57cec5SDimitry Andric report("inconsistent constant size", MI);
11590b57cec5SDimitry Andric } else {
11600b57cec5SDimitry Andric if (!MI->getOperand(1).isFPImm()) {
11610b57cec5SDimitry Andric report("G_FCONSTANT operand must be fpimm", MI);
11620b57cec5SDimitry Andric break;
11630b57cec5SDimitry Andric }
11640b57cec5SDimitry Andric const ConstantFP *CF = MI->getOperand(1).getFPImm();
11650b57cec5SDimitry Andric
11660b57cec5SDimitry Andric if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) !=
11670b57cec5SDimitry Andric DstTy.getSizeInBits()) {
11680b57cec5SDimitry Andric report("inconsistent constant size", MI);
11690b57cec5SDimitry Andric }
11700b57cec5SDimitry Andric }
11710b57cec5SDimitry Andric
11720b57cec5SDimitry Andric break;
11730b57cec5SDimitry Andric }
11740b57cec5SDimitry Andric case TargetOpcode::G_LOAD:
11750b57cec5SDimitry Andric case TargetOpcode::G_STORE:
11760b57cec5SDimitry Andric case TargetOpcode::G_ZEXTLOAD:
11770b57cec5SDimitry Andric case TargetOpcode::G_SEXTLOAD: {
11780b57cec5SDimitry Andric LLT ValTy = MRI->getType(MI->getOperand(0).getReg());
11790b57cec5SDimitry Andric LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());
11800b57cec5SDimitry Andric if (!PtrTy.isPointer())
11810b57cec5SDimitry Andric report("Generic memory instruction must access a pointer", MI);
11820b57cec5SDimitry Andric
11830b57cec5SDimitry Andric // Generic loads and stores must have a single MachineMemOperand
11840b57cec5SDimitry Andric // describing that access.
11850b57cec5SDimitry Andric if (!MI->hasOneMemOperand()) {
11860b57cec5SDimitry Andric report("Generic instruction accessing memory must have one mem operand",
11870b57cec5SDimitry Andric MI);
11880b57cec5SDimitry Andric } else {
11890b57cec5SDimitry Andric const MachineMemOperand &MMO = **MI->memoperands_begin();
11900b57cec5SDimitry Andric if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
11910b57cec5SDimitry Andric MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {
11920b57cec5SDimitry Andric if (MMO.getSizeInBits() >= ValTy.getSizeInBits())
11930b57cec5SDimitry Andric report("Generic extload must have a narrower memory type", MI);
11940b57cec5SDimitry Andric } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
11950b57cec5SDimitry Andric if (MMO.getSize() > ValTy.getSizeInBytes())
11960b57cec5SDimitry Andric report("load memory size cannot exceed result size", MI);
11970b57cec5SDimitry Andric } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
11980b57cec5SDimitry Andric if (ValTy.getSizeInBytes() < MMO.getSize())
11990b57cec5SDimitry Andric report("store memory size cannot exceed value size", MI);
12000b57cec5SDimitry Andric }
120181ad6265SDimitry Andric
120281ad6265SDimitry Andric const AtomicOrdering Order = MMO.getSuccessOrdering();
120381ad6265SDimitry Andric if (Opc == TargetOpcode::G_STORE) {
120481ad6265SDimitry Andric if (Order == AtomicOrdering::Acquire ||
120581ad6265SDimitry Andric Order == AtomicOrdering::AcquireRelease)
120681ad6265SDimitry Andric report("atomic store cannot use acquire ordering", MI);
120781ad6265SDimitry Andric
120881ad6265SDimitry Andric } else {
120981ad6265SDimitry Andric if (Order == AtomicOrdering::Release ||
121081ad6265SDimitry Andric Order == AtomicOrdering::AcquireRelease)
121181ad6265SDimitry Andric report("atomic load cannot use release ordering", MI);
121281ad6265SDimitry Andric }
12130b57cec5SDimitry Andric }
12140b57cec5SDimitry Andric
12150b57cec5SDimitry Andric break;
12160b57cec5SDimitry Andric }
12170b57cec5SDimitry Andric case TargetOpcode::G_PHI: {
12180b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1219e8d8bef9SDimitry Andric if (!DstTy.isValid() || !all_of(drop_begin(MI->operands()),
12200b57cec5SDimitry Andric [this, &DstTy](const MachineOperand &MO) {
12210b57cec5SDimitry Andric if (!MO.isReg())
12220b57cec5SDimitry Andric return true;
12230b57cec5SDimitry Andric LLT Ty = MRI->getType(MO.getReg());
12240b57cec5SDimitry Andric if (!Ty.isValid() || (Ty != DstTy))
12250b57cec5SDimitry Andric return false;
12260b57cec5SDimitry Andric return true;
12270b57cec5SDimitry Andric }))
12280b57cec5SDimitry Andric report("Generic Instruction G_PHI has operands with incompatible/missing "
12290b57cec5SDimitry Andric "types",
12300b57cec5SDimitry Andric MI);
12310b57cec5SDimitry Andric break;
12320b57cec5SDimitry Andric }
12330b57cec5SDimitry Andric case TargetOpcode::G_BITCAST: {
12340b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
12350b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
12360b57cec5SDimitry Andric if (!DstTy.isValid() || !SrcTy.isValid())
12370b57cec5SDimitry Andric break;
12380b57cec5SDimitry Andric
12390b57cec5SDimitry Andric if (SrcTy.isPointer() != DstTy.isPointer())
12400b57cec5SDimitry Andric report("bitcast cannot convert between pointers and other types", MI);
12410b57cec5SDimitry Andric
12420b57cec5SDimitry Andric if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
12430b57cec5SDimitry Andric report("bitcast sizes must match", MI);
12445ffd83dbSDimitry Andric
12455ffd83dbSDimitry Andric if (SrcTy == DstTy)
12465ffd83dbSDimitry Andric report("bitcast must change the type", MI);
12475ffd83dbSDimitry Andric
12480b57cec5SDimitry Andric break;
12490b57cec5SDimitry Andric }
12500b57cec5SDimitry Andric case TargetOpcode::G_INTTOPTR:
12510b57cec5SDimitry Andric case TargetOpcode::G_PTRTOINT:
12520b57cec5SDimitry Andric case TargetOpcode::G_ADDRSPACE_CAST: {
12530b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
12540b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
12550b57cec5SDimitry Andric if (!DstTy.isValid() || !SrcTy.isValid())
12560b57cec5SDimitry Andric break;
12570b57cec5SDimitry Andric
12580b57cec5SDimitry Andric verifyVectorElementMatch(DstTy, SrcTy, MI);
12590b57cec5SDimitry Andric
12600b57cec5SDimitry Andric DstTy = DstTy.getScalarType();
12610b57cec5SDimitry Andric SrcTy = SrcTy.getScalarType();
12620b57cec5SDimitry Andric
12630b57cec5SDimitry Andric if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
12640b57cec5SDimitry Andric if (!DstTy.isPointer())
12650b57cec5SDimitry Andric report("inttoptr result type must be a pointer", MI);
12660b57cec5SDimitry Andric if (SrcTy.isPointer())
12670b57cec5SDimitry Andric report("inttoptr source type must not be a pointer", MI);
12680b57cec5SDimitry Andric } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
12690b57cec5SDimitry Andric if (!SrcTy.isPointer())
12700b57cec5SDimitry Andric report("ptrtoint source type must be a pointer", MI);
12710b57cec5SDimitry Andric if (DstTy.isPointer())
12720b57cec5SDimitry Andric report("ptrtoint result type must not be a pointer", MI);
12730b57cec5SDimitry Andric } else {
12740b57cec5SDimitry Andric assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
12750b57cec5SDimitry Andric if (!SrcTy.isPointer() || !DstTy.isPointer())
12760b57cec5SDimitry Andric report("addrspacecast types must be pointers", MI);
12770b57cec5SDimitry Andric else {
12780b57cec5SDimitry Andric if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
12790b57cec5SDimitry Andric report("addrspacecast must convert different address spaces", MI);
12800b57cec5SDimitry Andric }
12810b57cec5SDimitry Andric }
12820b57cec5SDimitry Andric
12830b57cec5SDimitry Andric break;
12840b57cec5SDimitry Andric }
1285480093f4SDimitry Andric case TargetOpcode::G_PTR_ADD: {
12860b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
12870b57cec5SDimitry Andric LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());
12880b57cec5SDimitry Andric LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg());
12890b57cec5SDimitry Andric if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
12900b57cec5SDimitry Andric break;
12910b57cec5SDimitry Andric
12920b57cec5SDimitry Andric if (!PtrTy.getScalarType().isPointer())
12930b57cec5SDimitry Andric report("gep first operand must be a pointer", MI);
12940b57cec5SDimitry Andric
12950b57cec5SDimitry Andric if (OffsetTy.getScalarType().isPointer())
12960b57cec5SDimitry Andric report("gep offset operand must not be a pointer", MI);
12970b57cec5SDimitry Andric
12980b57cec5SDimitry Andric // TODO: Is the offset allowed to be a scalar with a vector?
12990b57cec5SDimitry Andric break;
13000b57cec5SDimitry Andric }
13015ffd83dbSDimitry Andric case TargetOpcode::G_PTRMASK: {
13025ffd83dbSDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
13035ffd83dbSDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
13045ffd83dbSDimitry Andric LLT MaskTy = MRI->getType(MI->getOperand(2).getReg());
13055ffd83dbSDimitry Andric if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
13065ffd83dbSDimitry Andric break;
13075ffd83dbSDimitry Andric
13085ffd83dbSDimitry Andric if (!DstTy.getScalarType().isPointer())
13095ffd83dbSDimitry Andric report("ptrmask result type must be a pointer", MI);
13105ffd83dbSDimitry Andric
13115ffd83dbSDimitry Andric if (!MaskTy.getScalarType().isScalar())
13125ffd83dbSDimitry Andric report("ptrmask mask type must be an integer", MI);
13135ffd83dbSDimitry Andric
13145ffd83dbSDimitry Andric verifyVectorElementMatch(DstTy, MaskTy, MI);
13155ffd83dbSDimitry Andric break;
13165ffd83dbSDimitry Andric }
13170b57cec5SDimitry Andric case TargetOpcode::G_SEXT:
13180b57cec5SDimitry Andric case TargetOpcode::G_ZEXT:
13190b57cec5SDimitry Andric case TargetOpcode::G_ANYEXT:
13200b57cec5SDimitry Andric case TargetOpcode::G_TRUNC:
13210b57cec5SDimitry Andric case TargetOpcode::G_FPEXT:
13220b57cec5SDimitry Andric case TargetOpcode::G_FPTRUNC: {
13230b57cec5SDimitry Andric // Number of operands and presense of types is already checked (and
13240b57cec5SDimitry Andric // reported in case of any issues), so no need to report them again. As
13250b57cec5SDimitry Andric // we're trying to report as many issues as possible at once, however, the
13260b57cec5SDimitry Andric // instructions aren't guaranteed to have the right number of operands or
13270b57cec5SDimitry Andric // types attached to them at this point
13280b57cec5SDimitry Andric assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
13290b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
13300b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
13310b57cec5SDimitry Andric if (!DstTy.isValid() || !SrcTy.isValid())
13320b57cec5SDimitry Andric break;
13330b57cec5SDimitry Andric
13340b57cec5SDimitry Andric LLT DstElTy = DstTy.getScalarType();
13350b57cec5SDimitry Andric LLT SrcElTy = SrcTy.getScalarType();
13360b57cec5SDimitry Andric if (DstElTy.isPointer() || SrcElTy.isPointer())
13370b57cec5SDimitry Andric report("Generic extend/truncate can not operate on pointers", MI);
13380b57cec5SDimitry Andric
13390b57cec5SDimitry Andric verifyVectorElementMatch(DstTy, SrcTy, MI);
13400b57cec5SDimitry Andric
13410b57cec5SDimitry Andric unsigned DstSize = DstElTy.getSizeInBits();
13420b57cec5SDimitry Andric unsigned SrcSize = SrcElTy.getSizeInBits();
13430b57cec5SDimitry Andric switch (MI->getOpcode()) {
13440b57cec5SDimitry Andric default:
13450b57cec5SDimitry Andric if (DstSize <= SrcSize)
13460b57cec5SDimitry Andric report("Generic extend has destination type no larger than source", MI);
13470b57cec5SDimitry Andric break;
13480b57cec5SDimitry Andric case TargetOpcode::G_TRUNC:
13490b57cec5SDimitry Andric case TargetOpcode::G_FPTRUNC:
13500b57cec5SDimitry Andric if (DstSize >= SrcSize)
13510b57cec5SDimitry Andric report("Generic truncate has destination type no smaller than source",
13520b57cec5SDimitry Andric MI);
13530b57cec5SDimitry Andric break;
13540b57cec5SDimitry Andric }
13550b57cec5SDimitry Andric break;
13560b57cec5SDimitry Andric }
13570b57cec5SDimitry Andric case TargetOpcode::G_SELECT: {
13580b57cec5SDimitry Andric LLT SelTy = MRI->getType(MI->getOperand(0).getReg());
13590b57cec5SDimitry Andric LLT CondTy = MRI->getType(MI->getOperand(1).getReg());
13600b57cec5SDimitry Andric if (!SelTy.isValid() || !CondTy.isValid())
13610b57cec5SDimitry Andric break;
13620b57cec5SDimitry Andric
13630b57cec5SDimitry Andric // Scalar condition select on a vector is valid.
13640b57cec5SDimitry Andric if (CondTy.isVector())
13650b57cec5SDimitry Andric verifyVectorElementMatch(SelTy, CondTy, MI);
13660b57cec5SDimitry Andric break;
13670b57cec5SDimitry Andric }
13680b57cec5SDimitry Andric case TargetOpcode::G_MERGE_VALUES: {
13690b57cec5SDimitry Andric // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
13700b57cec5SDimitry Andric // e.g. s2N = MERGE sN, sN
13710b57cec5SDimitry Andric // Merging multiple scalars into a vector is not allowed, should use
13720b57cec5SDimitry Andric // G_BUILD_VECTOR for that.
13730b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
13740b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
13750b57cec5SDimitry Andric if (DstTy.isVector() || SrcTy.isVector())
13760b57cec5SDimitry Andric report("G_MERGE_VALUES cannot operate on vectors", MI);
13770b57cec5SDimitry Andric
13780b57cec5SDimitry Andric const unsigned NumOps = MI->getNumOperands();
13790b57cec5SDimitry Andric if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
13800b57cec5SDimitry Andric report("G_MERGE_VALUES result size is inconsistent", MI);
13810b57cec5SDimitry Andric
13820b57cec5SDimitry Andric for (unsigned I = 2; I != NumOps; ++I) {
13830b57cec5SDimitry Andric if (MRI->getType(MI->getOperand(I).getReg()) != SrcTy)
13840b57cec5SDimitry Andric report("G_MERGE_VALUES source types do not match", MI);
13850b57cec5SDimitry Andric }
13860b57cec5SDimitry Andric
13870b57cec5SDimitry Andric break;
13880b57cec5SDimitry Andric }
13890b57cec5SDimitry Andric case TargetOpcode::G_UNMERGE_VALUES: {
1390bdd1243dSDimitry Andric unsigned NumDsts = MI->getNumOperands() - 1;
13910b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1392bdd1243dSDimitry Andric for (unsigned i = 1; i < NumDsts; ++i) {
1393bdd1243dSDimitry Andric if (MRI->getType(MI->getOperand(i).getReg()) != DstTy) {
13940b57cec5SDimitry Andric report("G_UNMERGE_VALUES destination types do not match", MI);
1395bdd1243dSDimitry Andric break;
13960b57cec5SDimitry Andric }
1397bdd1243dSDimitry Andric }
1398bdd1243dSDimitry Andric
1399bdd1243dSDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(NumDsts).getReg());
1400bdd1243dSDimitry Andric if (DstTy.isVector()) {
1401bdd1243dSDimitry Andric // This case is the converse of G_CONCAT_VECTORS.
1402bdd1243dSDimitry Andric if (!SrcTy.isVector() || SrcTy.getScalarType() != DstTy.getScalarType() ||
1403bdd1243dSDimitry Andric SrcTy.getNumElements() != NumDsts * DstTy.getNumElements())
1404bdd1243dSDimitry Andric report("G_UNMERGE_VALUES source operand does not match vector "
1405bdd1243dSDimitry Andric "destination operands",
14060b57cec5SDimitry Andric MI);
1407bdd1243dSDimitry Andric } else if (SrcTy.isVector()) {
1408bdd1243dSDimitry Andric // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1409bdd1243dSDimitry Andric // mismatched types as long as the total size matches:
1410bdd1243dSDimitry Andric // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1411bdd1243dSDimitry Andric if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1412bdd1243dSDimitry Andric report("G_UNMERGE_VALUES vector source operand does not match scalar "
1413bdd1243dSDimitry Andric "destination operands",
1414bdd1243dSDimitry Andric MI);
1415bdd1243dSDimitry Andric } else {
1416bdd1243dSDimitry Andric // This case is the converse of G_MERGE_VALUES.
1417bdd1243dSDimitry Andric if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1418bdd1243dSDimitry Andric report("G_UNMERGE_VALUES scalar source operand does not match scalar "
1419bdd1243dSDimitry Andric "destination operands",
1420bdd1243dSDimitry Andric MI);
1421bdd1243dSDimitry Andric }
14220b57cec5SDimitry Andric }
14230b57cec5SDimitry Andric break;
14240b57cec5SDimitry Andric }
14250b57cec5SDimitry Andric case TargetOpcode::G_BUILD_VECTOR: {
14260b57cec5SDimitry Andric // Source types must be scalars, dest type a vector. Total size of scalars
14270b57cec5SDimitry Andric // must match the dest vector size.
14280b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
14290b57cec5SDimitry Andric LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg());
14300b57cec5SDimitry Andric if (!DstTy.isVector() || SrcEltTy.isVector()) {
14310b57cec5SDimitry Andric report("G_BUILD_VECTOR must produce a vector from scalar operands", MI);
14320b57cec5SDimitry Andric break;
14330b57cec5SDimitry Andric }
14340b57cec5SDimitry Andric
14350b57cec5SDimitry Andric if (DstTy.getElementType() != SrcEltTy)
14360b57cec5SDimitry Andric report("G_BUILD_VECTOR result element type must match source type", MI);
14370b57cec5SDimitry Andric
14380b57cec5SDimitry Andric if (DstTy.getNumElements() != MI->getNumOperands() - 1)
14390b57cec5SDimitry Andric report("G_BUILD_VECTOR must have an operand for each elemement", MI);
14400b57cec5SDimitry Andric
14414824e7fdSDimitry Andric for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))
14424824e7fdSDimitry Andric if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))
14430b57cec5SDimitry Andric report("G_BUILD_VECTOR source operand types are not homogeneous", MI);
14440b57cec5SDimitry Andric
14450b57cec5SDimitry Andric break;
14460b57cec5SDimitry Andric }
14470b57cec5SDimitry Andric case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
14480b57cec5SDimitry Andric // Source types must be scalars, dest type a vector. Scalar types must be
14490b57cec5SDimitry Andric // larger than the dest vector elt type, as this is a truncating operation.
14500b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
14510b57cec5SDimitry Andric LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg());
14520b57cec5SDimitry Andric if (!DstTy.isVector() || SrcEltTy.isVector())
14530b57cec5SDimitry Andric report("G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
14540b57cec5SDimitry Andric MI);
14554824e7fdSDimitry Andric for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))
14564824e7fdSDimitry Andric if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))
14570b57cec5SDimitry Andric report("G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
14580b57cec5SDimitry Andric MI);
14590b57cec5SDimitry Andric if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
14600b57cec5SDimitry Andric report("G_BUILD_VECTOR_TRUNC source operand types are not larger than "
14610b57cec5SDimitry Andric "dest elt type",
14620b57cec5SDimitry Andric MI);
14630b57cec5SDimitry Andric break;
14640b57cec5SDimitry Andric }
14650b57cec5SDimitry Andric case TargetOpcode::G_CONCAT_VECTORS: {
14660b57cec5SDimitry Andric // Source types should be vectors, and total size should match the dest
14670b57cec5SDimitry Andric // vector size.
14680b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
14690b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
14700b57cec5SDimitry Andric if (!DstTy.isVector() || !SrcTy.isVector())
14710b57cec5SDimitry Andric report("G_CONCAT_VECTOR requires vector source and destination operands",
14720b57cec5SDimitry Andric MI);
1473fe6060f1SDimitry Andric
1474fe6060f1SDimitry Andric if (MI->getNumOperands() < 3)
1475fe6060f1SDimitry Andric report("G_CONCAT_VECTOR requires at least 2 source operands", MI);
1476fe6060f1SDimitry Andric
14774824e7fdSDimitry Andric for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))
14784824e7fdSDimitry Andric if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))
14790b57cec5SDimitry Andric report("G_CONCAT_VECTOR source operand types are not homogeneous", MI);
14800b57cec5SDimitry Andric if (DstTy.getNumElements() !=
14810b57cec5SDimitry Andric SrcTy.getNumElements() * (MI->getNumOperands() - 1))
14820b57cec5SDimitry Andric report("G_CONCAT_VECTOR num dest and source elements should match", MI);
14830b57cec5SDimitry Andric break;
14840b57cec5SDimitry Andric }
14850b57cec5SDimitry Andric case TargetOpcode::G_ICMP:
14860b57cec5SDimitry Andric case TargetOpcode::G_FCMP: {
14870b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
14880b57cec5SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(2).getReg());
14890b57cec5SDimitry Andric
14900b57cec5SDimitry Andric if ((DstTy.isVector() != SrcTy.isVector()) ||
14910b57cec5SDimitry Andric (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements()))
14920b57cec5SDimitry Andric report("Generic vector icmp/fcmp must preserve number of lanes", MI);
14930b57cec5SDimitry Andric
14940b57cec5SDimitry Andric break;
14950b57cec5SDimitry Andric }
14960b57cec5SDimitry Andric case TargetOpcode::G_EXTRACT: {
14970b57cec5SDimitry Andric const MachineOperand &SrcOp = MI->getOperand(1);
14980b57cec5SDimitry Andric if (!SrcOp.isReg()) {
14990b57cec5SDimitry Andric report("extract source must be a register", MI);
15000b57cec5SDimitry Andric break;
15010b57cec5SDimitry Andric }
15020b57cec5SDimitry Andric
15030b57cec5SDimitry Andric const MachineOperand &OffsetOp = MI->getOperand(2);
15040b57cec5SDimitry Andric if (!OffsetOp.isImm()) {
15050b57cec5SDimitry Andric report("extract offset must be a constant", MI);
15060b57cec5SDimitry Andric break;
15070b57cec5SDimitry Andric }
15080b57cec5SDimitry Andric
15090b57cec5SDimitry Andric unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits();
15100b57cec5SDimitry Andric unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits();
15110b57cec5SDimitry Andric if (SrcSize == DstSize)
15120b57cec5SDimitry Andric report("extract source must be larger than result", MI);
15130b57cec5SDimitry Andric
15140b57cec5SDimitry Andric if (DstSize + OffsetOp.getImm() > SrcSize)
15150b57cec5SDimitry Andric report("extract reads past end of register", MI);
15160b57cec5SDimitry Andric break;
15170b57cec5SDimitry Andric }
15180b57cec5SDimitry Andric case TargetOpcode::G_INSERT: {
15190b57cec5SDimitry Andric const MachineOperand &SrcOp = MI->getOperand(2);
15200b57cec5SDimitry Andric if (!SrcOp.isReg()) {
15210b57cec5SDimitry Andric report("insert source must be a register", MI);
15220b57cec5SDimitry Andric break;
15230b57cec5SDimitry Andric }
15240b57cec5SDimitry Andric
15250b57cec5SDimitry Andric const MachineOperand &OffsetOp = MI->getOperand(3);
15260b57cec5SDimitry Andric if (!OffsetOp.isImm()) {
15270b57cec5SDimitry Andric report("insert offset must be a constant", MI);
15280b57cec5SDimitry Andric break;
15290b57cec5SDimitry Andric }
15300b57cec5SDimitry Andric
15310b57cec5SDimitry Andric unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits();
15320b57cec5SDimitry Andric unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits();
15330b57cec5SDimitry Andric
15340b57cec5SDimitry Andric if (DstSize <= SrcSize)
15350b57cec5SDimitry Andric report("inserted size must be smaller than total register", MI);
15360b57cec5SDimitry Andric
15370b57cec5SDimitry Andric if (SrcSize + OffsetOp.getImm() > DstSize)
15380b57cec5SDimitry Andric report("insert writes past end of register", MI);
15390b57cec5SDimitry Andric
15400b57cec5SDimitry Andric break;
15410b57cec5SDimitry Andric }
15420b57cec5SDimitry Andric case TargetOpcode::G_JUMP_TABLE: {
15430b57cec5SDimitry Andric if (!MI->getOperand(1).isJTI())
15440b57cec5SDimitry Andric report("G_JUMP_TABLE source operand must be a jump table index", MI);
15450b57cec5SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
15460b57cec5SDimitry Andric if (!DstTy.isPointer())
15470b57cec5SDimitry Andric report("G_JUMP_TABLE dest operand must have a pointer type", MI);
15480b57cec5SDimitry Andric break;
15490b57cec5SDimitry Andric }
15500b57cec5SDimitry Andric case TargetOpcode::G_BRJT: {
15510b57cec5SDimitry Andric if (!MRI->getType(MI->getOperand(0).getReg()).isPointer())
15520b57cec5SDimitry Andric report("G_BRJT src operand 0 must be a pointer type", MI);
15530b57cec5SDimitry Andric
15540b57cec5SDimitry Andric if (!MI->getOperand(1).isJTI())
15550b57cec5SDimitry Andric report("G_BRJT src operand 1 must be a jump table index", MI);
15560b57cec5SDimitry Andric
15570b57cec5SDimitry Andric const auto &IdxOp = MI->getOperand(2);
15580b57cec5SDimitry Andric if (!IdxOp.isReg() || MRI->getType(IdxOp.getReg()).isPointer())
15590b57cec5SDimitry Andric report("G_BRJT src operand 2 must be a scalar reg type", MI);
15600b57cec5SDimitry Andric break;
15610b57cec5SDimitry Andric }
15620b57cec5SDimitry Andric case TargetOpcode::G_INTRINSIC:
15635f757f3fSDimitry Andric case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
15645f757f3fSDimitry Andric case TargetOpcode::G_INTRINSIC_CONVERGENT:
15655f757f3fSDimitry Andric case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
15660b57cec5SDimitry Andric // TODO: Should verify number of def and use operands, but the current
15670b57cec5SDimitry Andric // interface requires passing in IR types for mangling.
15680b57cec5SDimitry Andric const MachineOperand &IntrIDOp = MI->getOperand(MI->getNumExplicitDefs());
15690b57cec5SDimitry Andric if (!IntrIDOp.isIntrinsicID()) {
15700b57cec5SDimitry Andric report("G_INTRINSIC first src operand must be an intrinsic ID", MI);
15710b57cec5SDimitry Andric break;
15720b57cec5SDimitry Andric }
15730b57cec5SDimitry Andric
15745f757f3fSDimitry Andric if (!verifyGIntrinsicSideEffects(MI))
15750b57cec5SDimitry Andric break;
15765f757f3fSDimitry Andric if (!verifyGIntrinsicConvergence(MI))
15770b57cec5SDimitry Andric break;
1578e8d8bef9SDimitry Andric
15798bcb0991SDimitry Andric break;
15808bcb0991SDimitry Andric }
15818bcb0991SDimitry Andric case TargetOpcode::G_SEXT_INREG: {
15828bcb0991SDimitry Andric if (!MI->getOperand(2).isImm()) {
15838bcb0991SDimitry Andric report("G_SEXT_INREG expects an immediate operand #2", MI);
15848bcb0991SDimitry Andric break;
15858bcb0991SDimitry Andric }
15860b57cec5SDimitry Andric
15878bcb0991SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
15888bcb0991SDimitry Andric int64_t Imm = MI->getOperand(2).getImm();
15898bcb0991SDimitry Andric if (Imm <= 0)
15908bcb0991SDimitry Andric report("G_SEXT_INREG size must be >= 1", MI);
15918bcb0991SDimitry Andric if (Imm >= SrcTy.getScalarSizeInBits())
15928bcb0991SDimitry Andric report("G_SEXT_INREG size must be less than source bit width", MI);
15938bcb0991SDimitry Andric break;
15948bcb0991SDimitry Andric }
15955f757f3fSDimitry Andric case TargetOpcode::G_BSWAP: {
15965f757f3fSDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
15975f757f3fSDimitry Andric if (DstTy.getScalarSizeInBits() % 16 != 0)
15985f757f3fSDimitry Andric report("G_BSWAP size must be a multiple of 16 bits", MI);
15995f757f3fSDimitry Andric break;
16005f757f3fSDimitry Andric }
16018bcb0991SDimitry Andric case TargetOpcode::G_SHUFFLE_VECTOR: {
16028bcb0991SDimitry Andric const MachineOperand &MaskOp = MI->getOperand(3);
16038bcb0991SDimitry Andric if (!MaskOp.isShuffleMask()) {
16048bcb0991SDimitry Andric report("Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
16058bcb0991SDimitry Andric break;
16068bcb0991SDimitry Andric }
16078bcb0991SDimitry Andric
16088bcb0991SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
16098bcb0991SDimitry Andric LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg());
16108bcb0991SDimitry Andric LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg());
16118bcb0991SDimitry Andric
16128bcb0991SDimitry Andric if (Src0Ty != Src1Ty)
16138bcb0991SDimitry Andric report("Source operands must be the same type", MI);
16148bcb0991SDimitry Andric
16158bcb0991SDimitry Andric if (Src0Ty.getScalarType() != DstTy.getScalarType())
16168bcb0991SDimitry Andric report("G_SHUFFLE_VECTOR cannot change element type", MI);
16178bcb0991SDimitry Andric
16188bcb0991SDimitry Andric // Don't check that all operands are vector because scalars are used in
16198bcb0991SDimitry Andric // place of 1 element vectors.
16208bcb0991SDimitry Andric int SrcNumElts = Src0Ty.isVector() ? Src0Ty.getNumElements() : 1;
16218bcb0991SDimitry Andric int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
16228bcb0991SDimitry Andric
1623480093f4SDimitry Andric ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
16248bcb0991SDimitry Andric
16258bcb0991SDimitry Andric if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
16268bcb0991SDimitry Andric report("Wrong result type for shufflemask", MI);
16278bcb0991SDimitry Andric
16288bcb0991SDimitry Andric for (int Idx : MaskIdxes) {
16298bcb0991SDimitry Andric if (Idx < 0)
16308bcb0991SDimitry Andric continue;
16318bcb0991SDimitry Andric
16328bcb0991SDimitry Andric if (Idx >= 2 * SrcNumElts)
16338bcb0991SDimitry Andric report("Out of bounds shuffle index", MI);
16348bcb0991SDimitry Andric }
16358bcb0991SDimitry Andric
16368bcb0991SDimitry Andric break;
16378bcb0991SDimitry Andric }
16388bcb0991SDimitry Andric case TargetOpcode::G_DYN_STACKALLOC: {
16398bcb0991SDimitry Andric const MachineOperand &DstOp = MI->getOperand(0);
16408bcb0991SDimitry Andric const MachineOperand &AllocOp = MI->getOperand(1);
16418bcb0991SDimitry Andric const MachineOperand &AlignOp = MI->getOperand(2);
16428bcb0991SDimitry Andric
16438bcb0991SDimitry Andric if (!DstOp.isReg() || !MRI->getType(DstOp.getReg()).isPointer()) {
16448bcb0991SDimitry Andric report("dst operand 0 must be a pointer type", MI);
16458bcb0991SDimitry Andric break;
16468bcb0991SDimitry Andric }
16478bcb0991SDimitry Andric
16488bcb0991SDimitry Andric if (!AllocOp.isReg() || !MRI->getType(AllocOp.getReg()).isScalar()) {
16498bcb0991SDimitry Andric report("src operand 1 must be a scalar reg type", MI);
16508bcb0991SDimitry Andric break;
16518bcb0991SDimitry Andric }
16528bcb0991SDimitry Andric
16538bcb0991SDimitry Andric if (!AlignOp.isImm()) {
16548bcb0991SDimitry Andric report("src operand 2 must be an immediate type", MI);
16558bcb0991SDimitry Andric break;
16568bcb0991SDimitry Andric }
16570b57cec5SDimitry Andric break;
16580b57cec5SDimitry Andric }
1659fe6060f1SDimitry Andric case TargetOpcode::G_MEMCPY_INLINE:
1660e8d8bef9SDimitry Andric case TargetOpcode::G_MEMCPY:
1661e8d8bef9SDimitry Andric case TargetOpcode::G_MEMMOVE: {
1662e8d8bef9SDimitry Andric ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
1663e8d8bef9SDimitry Andric if (MMOs.size() != 2) {
1664e8d8bef9SDimitry Andric report("memcpy/memmove must have 2 memory operands", MI);
1665e8d8bef9SDimitry Andric break;
1666e8d8bef9SDimitry Andric }
1667e8d8bef9SDimitry Andric
1668e8d8bef9SDimitry Andric if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
1669e8d8bef9SDimitry Andric (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
1670e8d8bef9SDimitry Andric report("wrong memory operand types", MI);
1671e8d8bef9SDimitry Andric break;
1672e8d8bef9SDimitry Andric }
1673e8d8bef9SDimitry Andric
1674e8d8bef9SDimitry Andric if (MMOs[0]->getSize() != MMOs[1]->getSize())
1675e8d8bef9SDimitry Andric report("inconsistent memory operand sizes", MI);
1676e8d8bef9SDimitry Andric
1677e8d8bef9SDimitry Andric LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());
1678e8d8bef9SDimitry Andric LLT SrcPtrTy = MRI->getType(MI->getOperand(1).getReg());
1679e8d8bef9SDimitry Andric
1680e8d8bef9SDimitry Andric if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
1681e8d8bef9SDimitry Andric report("memory instruction operand must be a pointer", MI);
1682e8d8bef9SDimitry Andric break;
1683e8d8bef9SDimitry Andric }
1684e8d8bef9SDimitry Andric
1685e8d8bef9SDimitry Andric if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
1686e8d8bef9SDimitry Andric report("inconsistent store address space", MI);
1687e8d8bef9SDimitry Andric if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
1688e8d8bef9SDimitry Andric report("inconsistent load address space", MI);
1689e8d8bef9SDimitry Andric
1690fe6060f1SDimitry Andric if (Opc != TargetOpcode::G_MEMCPY_INLINE)
1691fe6060f1SDimitry Andric if (!MI->getOperand(3).isImm() || (MI->getOperand(3).getImm() & ~1LL))
1692fe6060f1SDimitry Andric report("'tail' flag (operand 3) must be an immediate 0 or 1", MI);
1693fe6060f1SDimitry Andric
1694e8d8bef9SDimitry Andric break;
1695e8d8bef9SDimitry Andric }
1696fe6060f1SDimitry Andric case TargetOpcode::G_BZERO:
1697e8d8bef9SDimitry Andric case TargetOpcode::G_MEMSET: {
1698e8d8bef9SDimitry Andric ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
1699fe6060f1SDimitry Andric std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
1700e8d8bef9SDimitry Andric if (MMOs.size() != 1) {
1701fe6060f1SDimitry Andric report(Twine(Name, " must have 1 memory operand"), MI);
1702e8d8bef9SDimitry Andric break;
1703e8d8bef9SDimitry Andric }
1704e8d8bef9SDimitry Andric
1705e8d8bef9SDimitry Andric if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
1706fe6060f1SDimitry Andric report(Twine(Name, " memory operand must be a store"), MI);
1707e8d8bef9SDimitry Andric break;
1708e8d8bef9SDimitry Andric }
1709e8d8bef9SDimitry Andric
1710e8d8bef9SDimitry Andric LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());
1711e8d8bef9SDimitry Andric if (!DstPtrTy.isPointer()) {
1712fe6060f1SDimitry Andric report(Twine(Name, " operand must be a pointer"), MI);
1713e8d8bef9SDimitry Andric break;
1714e8d8bef9SDimitry Andric }
1715e8d8bef9SDimitry Andric
1716e8d8bef9SDimitry Andric if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
1717fe6060f1SDimitry Andric report("inconsistent " + Twine(Name, " address space"), MI);
1718fe6060f1SDimitry Andric
1719fe6060f1SDimitry Andric if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||
1720fe6060f1SDimitry Andric (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))
1721fe6060f1SDimitry Andric report("'tail' flag (last operand) must be an immediate 0 or 1", MI);
1722e8d8bef9SDimitry Andric
1723e8d8bef9SDimitry Andric break;
1724e8d8bef9SDimitry Andric }
1725e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_SEQ_FADD:
1726e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
1727e8d8bef9SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1728e8d8bef9SDimitry Andric LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());
1729e8d8bef9SDimitry Andric LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());
1730e8d8bef9SDimitry Andric if (!DstTy.isScalar())
1731e8d8bef9SDimitry Andric report("Vector reduction requires a scalar destination type", MI);
1732e8d8bef9SDimitry Andric if (!Src1Ty.isScalar())
1733e8d8bef9SDimitry Andric report("Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
1734e8d8bef9SDimitry Andric if (!Src2Ty.isVector())
1735e8d8bef9SDimitry Andric report("Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
1736e8d8bef9SDimitry Andric break;
1737e8d8bef9SDimitry Andric }
1738e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_FADD:
1739e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_FMUL:
1740e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_FMAX:
1741e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_FMIN:
17425f757f3fSDimitry Andric case TargetOpcode::G_VECREDUCE_FMAXIMUM:
17435f757f3fSDimitry Andric case TargetOpcode::G_VECREDUCE_FMINIMUM:
1744e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_ADD:
1745e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_MUL:
1746e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_AND:
1747e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_OR:
1748e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_XOR:
1749e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_SMAX:
1750e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_SMIN:
1751e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_UMAX:
1752e8d8bef9SDimitry Andric case TargetOpcode::G_VECREDUCE_UMIN: {
1753e8d8bef9SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1754e8d8bef9SDimitry Andric if (!DstTy.isScalar())
1755e8d8bef9SDimitry Andric report("Vector reduction requires a scalar destination type", MI);
1756e8d8bef9SDimitry Andric break;
1757e8d8bef9SDimitry Andric }
1758fe6060f1SDimitry Andric
1759fe6060f1SDimitry Andric case TargetOpcode::G_SBFX:
1760fe6060f1SDimitry Andric case TargetOpcode::G_UBFX: {
1761fe6060f1SDimitry Andric LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1762fe6060f1SDimitry Andric if (DstTy.isVector()) {
1763fe6060f1SDimitry Andric report("Bitfield extraction is not supported on vectors", MI);
1764fe6060f1SDimitry Andric break;
1765fe6060f1SDimitry Andric }
1766fe6060f1SDimitry Andric break;
1767fe6060f1SDimitry Andric }
17680eae32dcSDimitry Andric case TargetOpcode::G_SHL:
17690eae32dcSDimitry Andric case TargetOpcode::G_LSHR:
17700eae32dcSDimitry Andric case TargetOpcode::G_ASHR:
1771fe6060f1SDimitry Andric case TargetOpcode::G_ROTR:
1772fe6060f1SDimitry Andric case TargetOpcode::G_ROTL: {
1773fe6060f1SDimitry Andric LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());
1774fe6060f1SDimitry Andric LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());
1775fe6060f1SDimitry Andric if (Src1Ty.isVector() != Src2Ty.isVector()) {
17760eae32dcSDimitry Andric report("Shifts and rotates require operands to be either all scalars or "
17770eae32dcSDimitry Andric "all vectors",
1778fe6060f1SDimitry Andric MI);
1779fe6060f1SDimitry Andric break;
1780fe6060f1SDimitry Andric }
1781fe6060f1SDimitry Andric break;
1782fe6060f1SDimitry Andric }
1783349cc55cSDimitry Andric case TargetOpcode::G_LLROUND:
1784349cc55cSDimitry Andric case TargetOpcode::G_LROUND: {
1785349cc55cSDimitry Andric verifyAllRegOpsScalar(*MI, *MRI);
1786349cc55cSDimitry Andric break;
1787349cc55cSDimitry Andric }
178881ad6265SDimitry Andric case TargetOpcode::G_IS_FPCLASS: {
178981ad6265SDimitry Andric LLT DestTy = MRI->getType(MI->getOperand(0).getReg());
179081ad6265SDimitry Andric LLT DestEltTy = DestTy.getScalarType();
179181ad6265SDimitry Andric if (!DestEltTy.isScalar()) {
179281ad6265SDimitry Andric report("Destination must be a scalar or vector of scalars", MI);
179381ad6265SDimitry Andric break;
179481ad6265SDimitry Andric }
179581ad6265SDimitry Andric LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
179681ad6265SDimitry Andric LLT SrcEltTy = SrcTy.getScalarType();
179781ad6265SDimitry Andric if (!SrcEltTy.isScalar()) {
179881ad6265SDimitry Andric report("Source must be a scalar or vector of scalars", MI);
179981ad6265SDimitry Andric break;
180081ad6265SDimitry Andric }
180181ad6265SDimitry Andric if (!verifyVectorElementMatch(DestTy, SrcTy, MI))
180281ad6265SDimitry Andric break;
180381ad6265SDimitry Andric const MachineOperand &TestMO = MI->getOperand(2);
180481ad6265SDimitry Andric if (!TestMO.isImm()) {
180581ad6265SDimitry Andric report("floating-point class set (operand 2) must be an immediate", MI);
180681ad6265SDimitry Andric break;
180781ad6265SDimitry Andric }
180881ad6265SDimitry Andric int64_t Test = TestMO.getImm();
180981ad6265SDimitry Andric if (Test < 0 || Test > fcAllFlags) {
181081ad6265SDimitry Andric report("Incorrect floating-point class set (operand 2)", MI);
181181ad6265SDimitry Andric break;
181281ad6265SDimitry Andric }
181381ad6265SDimitry Andric break;
181481ad6265SDimitry Andric }
18155f757f3fSDimitry Andric case TargetOpcode::G_PREFETCH: {
18165f757f3fSDimitry Andric const MachineOperand &AddrOp = MI->getOperand(0);
18175f757f3fSDimitry Andric if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) {
18185f757f3fSDimitry Andric report("addr operand must be a pointer", &AddrOp, 0);
18195f757f3fSDimitry Andric break;
18205f757f3fSDimitry Andric }
18215f757f3fSDimitry Andric const MachineOperand &RWOp = MI->getOperand(1);
18225f757f3fSDimitry Andric if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
18235f757f3fSDimitry Andric report("rw operand must be an immediate 0-1", &RWOp, 1);
18245f757f3fSDimitry Andric break;
18255f757f3fSDimitry Andric }
18265f757f3fSDimitry Andric const MachineOperand &LocalityOp = MI->getOperand(2);
18275f757f3fSDimitry Andric if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
18285f757f3fSDimitry Andric report("locality operand must be an immediate 0-3", &LocalityOp, 2);
18295f757f3fSDimitry Andric break;
18305f757f3fSDimitry Andric }
18315f757f3fSDimitry Andric const MachineOperand &CacheTypeOp = MI->getOperand(3);
18325f757f3fSDimitry Andric if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
18335f757f3fSDimitry Andric report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3);
18345f757f3fSDimitry Andric break;
18355f757f3fSDimitry Andric }
18365f757f3fSDimitry Andric break;
18375f757f3fSDimitry Andric }
1838bdd1243dSDimitry Andric case TargetOpcode::G_ASSERT_ALIGN: {
1839bdd1243dSDimitry Andric if (MI->getOperand(2).getImm() < 1)
1840bdd1243dSDimitry Andric report("alignment immediate must be >= 1", MI);
184181ad6265SDimitry Andric break;
184281ad6265SDimitry Andric }
184306c3fb27SDimitry Andric case TargetOpcode::G_CONSTANT_POOL: {
184406c3fb27SDimitry Andric if (!MI->getOperand(1).isCPI())
184506c3fb27SDimitry Andric report("Src operand 1 must be a constant pool index", MI);
184606c3fb27SDimitry Andric if (!MRI->getType(MI->getOperand(0).getReg()).isPointer())
184706c3fb27SDimitry Andric report("Dst operand 0 must be a pointer", MI);
184806c3fb27SDimitry Andric break;
184906c3fb27SDimitry Andric }
18500b57cec5SDimitry Andric default:
18510b57cec5SDimitry Andric break;
18520b57cec5SDimitry Andric }
18530b57cec5SDimitry Andric }
18540b57cec5SDimitry Andric
visitMachineInstrBefore(const MachineInstr * MI)18550b57cec5SDimitry Andric void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
18560b57cec5SDimitry Andric const MCInstrDesc &MCID = MI->getDesc();
18570b57cec5SDimitry Andric if (MI->getNumOperands() < MCID.getNumOperands()) {
18580b57cec5SDimitry Andric report("Too few operands", MI);
18590b57cec5SDimitry Andric errs() << MCID.getNumOperands() << " operands expected, but "
18600b57cec5SDimitry Andric << MI->getNumOperands() << " given.\n";
18610b57cec5SDimitry Andric }
18620b57cec5SDimitry Andric
18635f757f3fSDimitry Andric if (MI->getFlag(MachineInstr::NoConvergent) && !MCID.isConvergent())
18645f757f3fSDimitry Andric report("NoConvergent flag expected only on convergent instructions.", MI);
18655f757f3fSDimitry Andric
18660b57cec5SDimitry Andric if (MI->isPHI()) {
18670b57cec5SDimitry Andric if (MF->getProperties().hasProperty(
18680b57cec5SDimitry Andric MachineFunctionProperties::Property::NoPHIs))
18690b57cec5SDimitry Andric report("Found PHI instruction with NoPHIs property set", MI);
18700b57cec5SDimitry Andric
18710b57cec5SDimitry Andric if (FirstNonPHI)
18720b57cec5SDimitry Andric report("Found PHI instruction after non-PHI", MI);
18730b57cec5SDimitry Andric } else if (FirstNonPHI == nullptr)
18740b57cec5SDimitry Andric FirstNonPHI = MI;
18750b57cec5SDimitry Andric
18760b57cec5SDimitry Andric // Check the tied operands.
18770b57cec5SDimitry Andric if (MI->isInlineAsm())
18780b57cec5SDimitry Andric verifyInlineAsm(MI);
18790b57cec5SDimitry Andric
1880e8d8bef9SDimitry Andric // Check that unspillable terminators define a reg and have at most one use.
1881e8d8bef9SDimitry Andric if (TII->isUnspillableTerminator(MI)) {
1882e8d8bef9SDimitry Andric if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef())
1883e8d8bef9SDimitry Andric report("Unspillable Terminator does not define a reg", MI);
1884e8d8bef9SDimitry Andric Register Def = MI->getOperand(0).getReg();
1885e8d8bef9SDimitry Andric if (Def.isVirtual() &&
1886349cc55cSDimitry Andric !MF->getProperties().hasProperty(
1887349cc55cSDimitry Andric MachineFunctionProperties::Property::NoPHIs) &&
1888e8d8bef9SDimitry Andric std::distance(MRI->use_nodbg_begin(Def), MRI->use_nodbg_end()) > 1)
1889e8d8bef9SDimitry Andric report("Unspillable Terminator expected to have at most one use!", MI);
1890e8d8bef9SDimitry Andric }
1891e8d8bef9SDimitry Andric
18925ffd83dbSDimitry Andric // A fully-formed DBG_VALUE must have a location. Ignore partially formed
18935ffd83dbSDimitry Andric // DBG_VALUEs: these are convenient to use in tests, but should never get
18945ffd83dbSDimitry Andric // generated.
18955ffd83dbSDimitry Andric if (MI->isDebugValue() && MI->getNumOperands() == 4)
18965ffd83dbSDimitry Andric if (!MI->getDebugLoc())
18975ffd83dbSDimitry Andric report("Missing DebugLoc for debug instruction", MI);
18985ffd83dbSDimitry Andric
1899e8d8bef9SDimitry Andric // Meta instructions should never be the subject of debug value tracking,
1900e8d8bef9SDimitry Andric // they don't create a value in the output program at all.
1901e8d8bef9SDimitry Andric if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
1902e8d8bef9SDimitry Andric report("Metadata instruction should not have a value tracking number", MI);
1903e8d8bef9SDimitry Andric
19040b57cec5SDimitry Andric // Check the MachineMemOperands for basic consistency.
19055ffd83dbSDimitry Andric for (MachineMemOperand *Op : MI->memoperands()) {
19065ffd83dbSDimitry Andric if (Op->isLoad() && !MI->mayLoad())
19070b57cec5SDimitry Andric report("Missing mayLoad flag", MI);
19085ffd83dbSDimitry Andric if (Op->isStore() && !MI->mayStore())
19090b57cec5SDimitry Andric report("Missing mayStore flag", MI);
19100b57cec5SDimitry Andric }
19110b57cec5SDimitry Andric
19120b57cec5SDimitry Andric // Debug values must not have a slot index.
19130b57cec5SDimitry Andric // Other instructions must have one, unless they are inside a bundle.
19140b57cec5SDimitry Andric if (LiveInts) {
19150b57cec5SDimitry Andric bool mapped = !LiveInts->isNotInMIMap(*MI);
1916fe6060f1SDimitry Andric if (MI->isDebugOrPseudoInstr()) {
19170b57cec5SDimitry Andric if (mapped)
19180b57cec5SDimitry Andric report("Debug instruction has a slot index", MI);
19190b57cec5SDimitry Andric } else if (MI->isInsideBundle()) {
19200b57cec5SDimitry Andric if (mapped)
19210b57cec5SDimitry Andric report("Instruction inside bundle has a slot index", MI);
19220b57cec5SDimitry Andric } else {
19230b57cec5SDimitry Andric if (!mapped)
19240b57cec5SDimitry Andric report("Missing slot index", MI);
19250b57cec5SDimitry Andric }
19260b57cec5SDimitry Andric }
19270b57cec5SDimitry Andric
1928fe6060f1SDimitry Andric unsigned Opc = MCID.getOpcode();
1929fe6060f1SDimitry Andric if (isPreISelGenericOpcode(Opc) || isPreISelGenericOptimizationHint(Opc)) {
19300b57cec5SDimitry Andric verifyPreISelGenericInstruction(MI);
19310b57cec5SDimitry Andric return;
19320b57cec5SDimitry Andric }
19330b57cec5SDimitry Andric
19340b57cec5SDimitry Andric StringRef ErrorInfo;
19350b57cec5SDimitry Andric if (!TII->verifyInstruction(*MI, ErrorInfo))
19360b57cec5SDimitry Andric report(ErrorInfo.data(), MI);
19370b57cec5SDimitry Andric
19380b57cec5SDimitry Andric // Verify properties of various specific instruction types
19390b57cec5SDimitry Andric switch (MI->getOpcode()) {
19400b57cec5SDimitry Andric case TargetOpcode::COPY: {
19410b57cec5SDimitry Andric const MachineOperand &DstOp = MI->getOperand(0);
19420b57cec5SDimitry Andric const MachineOperand &SrcOp = MI->getOperand(1);
1943fe6060f1SDimitry Andric const Register SrcReg = SrcOp.getReg();
1944fe6060f1SDimitry Andric const Register DstReg = DstOp.getReg();
1945fe6060f1SDimitry Andric
1946fe6060f1SDimitry Andric LLT DstTy = MRI->getType(DstReg);
1947fe6060f1SDimitry Andric LLT SrcTy = MRI->getType(SrcReg);
19480b57cec5SDimitry Andric if (SrcTy.isValid() && DstTy.isValid()) {
19490b57cec5SDimitry Andric // If both types are valid, check that the types are the same.
19500b57cec5SDimitry Andric if (SrcTy != DstTy) {
19510b57cec5SDimitry Andric report("Copy Instruction is illegal with mismatching types", MI);
19520b57cec5SDimitry Andric errs() << "Def = " << DstTy << ", Src = " << SrcTy << "\n";
19530b57cec5SDimitry Andric }
1954fe6060f1SDimitry Andric
1955fe6060f1SDimitry Andric break;
19560b57cec5SDimitry Andric }
1957fe6060f1SDimitry Andric
1958fe6060f1SDimitry Andric if (!SrcTy.isValid() && !DstTy.isValid())
1959fe6060f1SDimitry Andric break;
1960fe6060f1SDimitry Andric
1961fe6060f1SDimitry Andric // If we have only one valid type, this is likely a copy between a virtual
1962fe6060f1SDimitry Andric // and physical register.
19635f757f3fSDimitry Andric TypeSize SrcSize = TRI->getRegSizeInBits(SrcReg, *MRI);
19645f757f3fSDimitry Andric TypeSize DstSize = TRI->getRegSizeInBits(DstReg, *MRI);
1965fe6060f1SDimitry Andric if (SrcReg.isPhysical() && DstTy.isValid()) {
1966fe6060f1SDimitry Andric const TargetRegisterClass *SrcRC =
1967fe6060f1SDimitry Andric TRI->getMinimalPhysRegClassLLT(SrcReg, DstTy);
1968fe6060f1SDimitry Andric if (SrcRC)
1969fe6060f1SDimitry Andric SrcSize = TRI->getRegSizeInBits(*SrcRC);
1970fe6060f1SDimitry Andric }
1971fe6060f1SDimitry Andric
1972fe6060f1SDimitry Andric if (DstReg.isPhysical() && SrcTy.isValid()) {
1973fe6060f1SDimitry Andric const TargetRegisterClass *DstRC =
1974fe6060f1SDimitry Andric TRI->getMinimalPhysRegClassLLT(DstReg, SrcTy);
1975fe6060f1SDimitry Andric if (DstRC)
1976fe6060f1SDimitry Andric DstSize = TRI->getRegSizeInBits(*DstRC);
1977fe6060f1SDimitry Andric }
1978fe6060f1SDimitry Andric
19795f757f3fSDimitry Andric // The next two checks allow COPY between physical and virtual registers,
19805f757f3fSDimitry Andric // when the virtual register has a scalable size and the physical register
19815f757f3fSDimitry Andric // has a fixed size. These checks allow COPY between *potentialy* mismatched
19825f757f3fSDimitry Andric // sizes. However, once RegisterBankSelection occurs, MachineVerifier should
19835f757f3fSDimitry Andric // be able to resolve a fixed size for the scalable vector, and at that
19845f757f3fSDimitry Andric // point this function will know for sure whether the sizes are mismatched
19855f757f3fSDimitry Andric // and correctly report a size mismatch.
19865f757f3fSDimitry Andric if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
19875f757f3fSDimitry Andric !SrcSize.isScalable())
19885f757f3fSDimitry Andric break;
19895f757f3fSDimitry Andric if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
19905f757f3fSDimitry Andric !DstSize.isScalable())
19915f757f3fSDimitry Andric break;
1992fe6060f1SDimitry Andric
19935f757f3fSDimitry Andric if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
19940b57cec5SDimitry Andric if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
19950b57cec5SDimitry Andric report("Copy Instruction is illegal with mismatching sizes", MI);
19960b57cec5SDimitry Andric errs() << "Def Size = " << DstSize << ", Src Size = " << SrcSize
19970b57cec5SDimitry Andric << "\n";
19980b57cec5SDimitry Andric }
19990b57cec5SDimitry Andric }
20000b57cec5SDimitry Andric break;
20010b57cec5SDimitry Andric }
20025ffd83dbSDimitry Andric case TargetOpcode::STATEPOINT: {
20035ffd83dbSDimitry Andric StatepointOpers SO(MI);
20045ffd83dbSDimitry Andric if (!MI->getOperand(SO.getIDPos()).isImm() ||
20055ffd83dbSDimitry Andric !MI->getOperand(SO.getNBytesPos()).isImm() ||
20065ffd83dbSDimitry Andric !MI->getOperand(SO.getNCallArgsPos()).isImm()) {
20070b57cec5SDimitry Andric report("meta operands to STATEPOINT not constant!", MI);
20080b57cec5SDimitry Andric break;
20095ffd83dbSDimitry Andric }
20100b57cec5SDimitry Andric
20110b57cec5SDimitry Andric auto VerifyStackMapConstant = [&](unsigned Offset) {
2012e8d8bef9SDimitry Andric if (Offset >= MI->getNumOperands()) {
2013e8d8bef9SDimitry Andric report("stack map constant to STATEPOINT is out of range!", MI);
2014e8d8bef9SDimitry Andric return;
2015e8d8bef9SDimitry Andric }
20165ffd83dbSDimitry Andric if (!MI->getOperand(Offset - 1).isImm() ||
20175ffd83dbSDimitry Andric MI->getOperand(Offset - 1).getImm() != StackMaps::ConstantOp ||
20185ffd83dbSDimitry Andric !MI->getOperand(Offset).isImm())
20190b57cec5SDimitry Andric report("stack map constant to STATEPOINT not well formed!", MI);
20200b57cec5SDimitry Andric };
20215ffd83dbSDimitry Andric VerifyStackMapConstant(SO.getCCIdx());
20225ffd83dbSDimitry Andric VerifyStackMapConstant(SO.getFlagsIdx());
20235ffd83dbSDimitry Andric VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2024e8d8bef9SDimitry Andric VerifyStackMapConstant(SO.getNumGCPtrIdx());
2025e8d8bef9SDimitry Andric VerifyStackMapConstant(SO.getNumAllocaIdx());
2026e8d8bef9SDimitry Andric VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2027e8d8bef9SDimitry Andric
2028e8d8bef9SDimitry Andric // Verify that all explicit statepoint defs are tied to gc operands as
2029e8d8bef9SDimitry Andric // they are expected to be a relocation of gc operands.
2030e8d8bef9SDimitry Andric unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2031e8d8bef9SDimitry Andric unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2032e8d8bef9SDimitry Andric for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2033e8d8bef9SDimitry Andric unsigned UseOpIdx;
2034e8d8bef9SDimitry Andric if (!MI->isRegTiedToUseOperand(Idx, &UseOpIdx)) {
2035e8d8bef9SDimitry Andric report("STATEPOINT defs expected to be tied", MI);
2036e8d8bef9SDimitry Andric break;
2037e8d8bef9SDimitry Andric }
2038e8d8bef9SDimitry Andric if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2039e8d8bef9SDimitry Andric report("STATEPOINT def tied to non-gc operand", MI);
2040e8d8bef9SDimitry Andric break;
2041e8d8bef9SDimitry Andric }
2042e8d8bef9SDimitry Andric }
20430b57cec5SDimitry Andric
20440b57cec5SDimitry Andric // TODO: verify we have properly encoded deopt arguments
20455ffd83dbSDimitry Andric } break;
2046fe6060f1SDimitry Andric case TargetOpcode::INSERT_SUBREG: {
2047fe6060f1SDimitry Andric unsigned InsertedSize;
2048fe6060f1SDimitry Andric if (unsigned SubIdx = MI->getOperand(2).getSubReg())
2049fe6060f1SDimitry Andric InsertedSize = TRI->getSubRegIdxSize(SubIdx);
2050fe6060f1SDimitry Andric else
2051fe6060f1SDimitry Andric InsertedSize = TRI->getRegSizeInBits(MI->getOperand(2).getReg(), *MRI);
2052fe6060f1SDimitry Andric unsigned SubRegSize = TRI->getSubRegIdxSize(MI->getOperand(3).getImm());
2053fe6060f1SDimitry Andric if (SubRegSize < InsertedSize) {
2054fe6060f1SDimitry Andric report("INSERT_SUBREG expected inserted value to have equal or lesser "
2055fe6060f1SDimitry Andric "size than the subreg it was inserted into", MI);
2056fe6060f1SDimitry Andric break;
2057fe6060f1SDimitry Andric }
2058fe6060f1SDimitry Andric } break;
2059bdd1243dSDimitry Andric case TargetOpcode::REG_SEQUENCE: {
2060bdd1243dSDimitry Andric unsigned NumOps = MI->getNumOperands();
2061bdd1243dSDimitry Andric if (!(NumOps & 1)) {
2062bdd1243dSDimitry Andric report("Invalid number of operands for REG_SEQUENCE", MI);
2063bdd1243dSDimitry Andric break;
2064bdd1243dSDimitry Andric }
2065bdd1243dSDimitry Andric
2066bdd1243dSDimitry Andric for (unsigned I = 1; I != NumOps; I += 2) {
2067bdd1243dSDimitry Andric const MachineOperand &RegOp = MI->getOperand(I);
2068bdd1243dSDimitry Andric const MachineOperand &SubRegOp = MI->getOperand(I + 1);
2069bdd1243dSDimitry Andric
2070bdd1243dSDimitry Andric if (!RegOp.isReg())
2071bdd1243dSDimitry Andric report("Invalid register operand for REG_SEQUENCE", &RegOp, I);
2072bdd1243dSDimitry Andric
2073bdd1243dSDimitry Andric if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2074bdd1243dSDimitry Andric SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2075bdd1243dSDimitry Andric report("Invalid subregister index operand for REG_SEQUENCE",
2076bdd1243dSDimitry Andric &SubRegOp, I + 1);
2077bdd1243dSDimitry Andric }
2078bdd1243dSDimitry Andric }
2079bdd1243dSDimitry Andric
2080bdd1243dSDimitry Andric Register DstReg = MI->getOperand(0).getReg();
2081bdd1243dSDimitry Andric if (DstReg.isPhysical())
2082bdd1243dSDimitry Andric report("REG_SEQUENCE does not support physical register results", MI);
2083bdd1243dSDimitry Andric
2084bdd1243dSDimitry Andric if (MI->getOperand(0).getSubReg())
2085bdd1243dSDimitry Andric report("Invalid subreg result for REG_SEQUENCE", MI);
2086bdd1243dSDimitry Andric
2087bdd1243dSDimitry Andric break;
2088bdd1243dSDimitry Andric }
20890b57cec5SDimitry Andric }
20900b57cec5SDimitry Andric }
20910b57cec5SDimitry Andric
20920b57cec5SDimitry Andric void
visitMachineOperand(const MachineOperand * MO,unsigned MONum)20930b57cec5SDimitry Andric MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
20940b57cec5SDimitry Andric const MachineInstr *MI = MO->getParent();
20950b57cec5SDimitry Andric const MCInstrDesc &MCID = MI->getDesc();
20960b57cec5SDimitry Andric unsigned NumDefs = MCID.getNumDefs();
20970b57cec5SDimitry Andric if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
20980b57cec5SDimitry Andric NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
20990b57cec5SDimitry Andric
21000b57cec5SDimitry Andric // The first MCID.NumDefs operands must be explicit register defines
21010b57cec5SDimitry Andric if (MONum < NumDefs) {
2102bdd1243dSDimitry Andric const MCOperandInfo &MCOI = MCID.operands()[MONum];
21030b57cec5SDimitry Andric if (!MO->isReg())
21040b57cec5SDimitry Andric report("Explicit definition must be a register", MO, MONum);
21050b57cec5SDimitry Andric else if (!MO->isDef() && !MCOI.isOptionalDef())
21060b57cec5SDimitry Andric report("Explicit definition marked as use", MO, MONum);
21070b57cec5SDimitry Andric else if (MO->isImplicit())
21080b57cec5SDimitry Andric report("Explicit definition marked as implicit", MO, MONum);
21090b57cec5SDimitry Andric } else if (MONum < MCID.getNumOperands()) {
2110bdd1243dSDimitry Andric const MCOperandInfo &MCOI = MCID.operands()[MONum];
21110b57cec5SDimitry Andric // Don't check if it's the last operand in a variadic instruction. See,
2112480093f4SDimitry Andric // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2113480093f4SDimitry Andric bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2114480093f4SDimitry Andric if (!IsOptional) {
2115480093f4SDimitry Andric if (MO->isReg()) {
21165ffd83dbSDimitry Andric if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
21170b57cec5SDimitry Andric report("Explicit operand marked as def", MO, MONum);
21180b57cec5SDimitry Andric if (MO->isImplicit())
21190b57cec5SDimitry Andric report("Explicit operand marked as implicit", MO, MONum);
21200b57cec5SDimitry Andric }
21210b57cec5SDimitry Andric
2122480093f4SDimitry Andric // Check that an instruction has register operands only as expected.
2123480093f4SDimitry Andric if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2124480093f4SDimitry Andric !MO->isReg() && !MO->isFI())
2125480093f4SDimitry Andric report("Expected a register operand.", MO, MONum);
2126fe6060f1SDimitry Andric if (MO->isReg()) {
2127fe6060f1SDimitry Andric if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2128fe6060f1SDimitry Andric (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2129fe6060f1SDimitry Andric !TII->isPCRelRegisterOperandLegal(*MO)))
2130480093f4SDimitry Andric report("Expected a non-register operand.", MO, MONum);
2131480093f4SDimitry Andric }
2132fe6060f1SDimitry Andric }
2133480093f4SDimitry Andric
21340b57cec5SDimitry Andric int TiedTo = MCID.getOperandConstraint(MONum, MCOI::TIED_TO);
21350b57cec5SDimitry Andric if (TiedTo != -1) {
21360b57cec5SDimitry Andric if (!MO->isReg())
21370b57cec5SDimitry Andric report("Tied use must be a register", MO, MONum);
21380b57cec5SDimitry Andric else if (!MO->isTied())
21390b57cec5SDimitry Andric report("Operand should be tied", MO, MONum);
21400b57cec5SDimitry Andric else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum))
21410b57cec5SDimitry Andric report("Tied def doesn't match MCInstrDesc", MO, MONum);
2142bdd1243dSDimitry Andric else if (MO->getReg().isPhysical()) {
21430b57cec5SDimitry Andric const MachineOperand &MOTied = MI->getOperand(TiedTo);
21440b57cec5SDimitry Andric if (!MOTied.isReg())
21450b57cec5SDimitry Andric report("Tied counterpart must be a register", &MOTied, TiedTo);
2146bdd1243dSDimitry Andric else if (MOTied.getReg().isPhysical() &&
21470b57cec5SDimitry Andric MO->getReg() != MOTied.getReg())
21480b57cec5SDimitry Andric report("Tied physical registers must match.", &MOTied, TiedTo);
21490b57cec5SDimitry Andric }
21500b57cec5SDimitry Andric } else if (MO->isReg() && MO->isTied())
21510b57cec5SDimitry Andric report("Explicit operand should not be tied", MO, MONum);
21525f757f3fSDimitry Andric } else if (!MI->isVariadic()) {
21530b57cec5SDimitry Andric // ARM adds %reg0 operands to indicate predicates. We'll allow that.
21545f757f3fSDimitry Andric if (!MO->isValidExcessOperand())
21550b57cec5SDimitry Andric report("Extra explicit operand on non-variadic instruction", MO, MONum);
21560b57cec5SDimitry Andric }
21570b57cec5SDimitry Andric
21580b57cec5SDimitry Andric switch (MO->getType()) {
21590b57cec5SDimitry Andric case MachineOperand::MO_Register: {
2160349cc55cSDimitry Andric // Verify debug flag on debug instructions. Check this first because reg0
2161349cc55cSDimitry Andric // indicates an undefined debug value.
2162349cc55cSDimitry Andric if (MI->isDebugInstr() && MO->isUse()) {
2163349cc55cSDimitry Andric if (!MO->isDebug())
2164349cc55cSDimitry Andric report("Register operand must be marked debug", MO, MONum);
2165349cc55cSDimitry Andric } else if (MO->isDebug()) {
2166349cc55cSDimitry Andric report("Register operand must not be marked debug", MO, MONum);
2167349cc55cSDimitry Andric }
2168349cc55cSDimitry Andric
21698bcb0991SDimitry Andric const Register Reg = MO->getReg();
21700b57cec5SDimitry Andric if (!Reg)
21710b57cec5SDimitry Andric return;
21721fd87a68SDimitry Andric if (MRI->tracksLiveness() && !MI->isDebugInstr())
21730b57cec5SDimitry Andric checkLiveness(MO, MONum);
21740b57cec5SDimitry Andric
217581ad6265SDimitry Andric if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
217681ad6265SDimitry Andric MO->getReg().isVirtual()) // TODO: Apply to physregs too
217781ad6265SDimitry Andric report("Undef virtual register def operands require a subregister", MO, MONum);
217881ad6265SDimitry Andric
21790b57cec5SDimitry Andric // Verify the consistency of tied operands.
21800b57cec5SDimitry Andric if (MO->isTied()) {
21810b57cec5SDimitry Andric unsigned OtherIdx = MI->findTiedOperandIdx(MONum);
21820b57cec5SDimitry Andric const MachineOperand &OtherMO = MI->getOperand(OtherIdx);
21830b57cec5SDimitry Andric if (!OtherMO.isReg())
21840b57cec5SDimitry Andric report("Must be tied to a register", MO, MONum);
21850b57cec5SDimitry Andric if (!OtherMO.isTied())
21860b57cec5SDimitry Andric report("Missing tie flags on tied operand", MO, MONum);
21870b57cec5SDimitry Andric if (MI->findTiedOperandIdx(OtherIdx) != MONum)
21880b57cec5SDimitry Andric report("Inconsistent tie links", MO, MONum);
21890b57cec5SDimitry Andric if (MONum < MCID.getNumDefs()) {
21900b57cec5SDimitry Andric if (OtherIdx < MCID.getNumOperands()) {
21910b57cec5SDimitry Andric if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO))
21920b57cec5SDimitry Andric report("Explicit def tied to explicit use without tie constraint",
21930b57cec5SDimitry Andric MO, MONum);
21940b57cec5SDimitry Andric } else {
21950b57cec5SDimitry Andric if (!OtherMO.isImplicit())
21960b57cec5SDimitry Andric report("Explicit def should be tied to implicit use", MO, MONum);
21970b57cec5SDimitry Andric }
21980b57cec5SDimitry Andric }
21990b57cec5SDimitry Andric }
22000b57cec5SDimitry Andric
22015ffd83dbSDimitry Andric // Verify two-address constraints after the twoaddressinstruction pass.
22025ffd83dbSDimitry Andric // Both twoaddressinstruction pass and phi-node-elimination pass call
22035f757f3fSDimitry Andric // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
22045f757f3fSDimitry Andric // after twoaddressinstruction pass not after phi-node-elimination pass. So
22055f757f3fSDimitry Andric // we shouldn't use the IsSSA as the condition, we should based on
22065ffd83dbSDimitry Andric // TiedOpsRewritten property to verify two-address constraints, this
22075ffd83dbSDimitry Andric // property will be set in twoaddressinstruction pass.
22080b57cec5SDimitry Andric unsigned DefIdx;
22095ffd83dbSDimitry Andric if (MF->getProperties().hasProperty(
22105ffd83dbSDimitry Andric MachineFunctionProperties::Property::TiedOpsRewritten) &&
22115ffd83dbSDimitry Andric MO->isUse() && MI->isRegTiedToDefOperand(MONum, &DefIdx) &&
22120b57cec5SDimitry Andric Reg != MI->getOperand(DefIdx).getReg())
22130b57cec5SDimitry Andric report("Two-address instruction operands must be identical", MO, MONum);
22140b57cec5SDimitry Andric
22150b57cec5SDimitry Andric // Check register classes.
22160b57cec5SDimitry Andric unsigned SubIdx = MO->getSubReg();
22170b57cec5SDimitry Andric
2218bdd1243dSDimitry Andric if (Reg.isPhysical()) {
22190b57cec5SDimitry Andric if (SubIdx) {
22200b57cec5SDimitry Andric report("Illegal subregister index for physical register", MO, MONum);
22210b57cec5SDimitry Andric return;
22220b57cec5SDimitry Andric }
22230b57cec5SDimitry Andric if (MONum < MCID.getNumOperands()) {
22240b57cec5SDimitry Andric if (const TargetRegisterClass *DRC =
22250b57cec5SDimitry Andric TII->getRegClass(MCID, MONum, TRI, *MF)) {
22260b57cec5SDimitry Andric if (!DRC->contains(Reg)) {
22270b57cec5SDimitry Andric report("Illegal physical register for instruction", MO, MONum);
22280b57cec5SDimitry Andric errs() << printReg(Reg, TRI) << " is not a "
22290b57cec5SDimitry Andric << TRI->getRegClassName(DRC) << " register.\n";
22300b57cec5SDimitry Andric }
22310b57cec5SDimitry Andric }
22320b57cec5SDimitry Andric }
22330b57cec5SDimitry Andric if (MO->isRenamable()) {
22340b57cec5SDimitry Andric if (MRI->isReserved(Reg)) {
22350b57cec5SDimitry Andric report("isRenamable set on reserved register", MO, MONum);
22360b57cec5SDimitry Andric return;
22370b57cec5SDimitry Andric }
22380b57cec5SDimitry Andric }
22390b57cec5SDimitry Andric } else {
22400b57cec5SDimitry Andric // Virtual register.
22410b57cec5SDimitry Andric const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
22420b57cec5SDimitry Andric if (!RC) {
22430b57cec5SDimitry Andric // This is a generic virtual register.
22440b57cec5SDimitry Andric
22455ffd83dbSDimitry Andric // Do not allow undef uses for generic virtual registers. This ensures
22465ffd83dbSDimitry Andric // getVRegDef can never fail and return null on a generic register.
22475ffd83dbSDimitry Andric //
22485ffd83dbSDimitry Andric // FIXME: This restriction should probably be broadened to all SSA
22495ffd83dbSDimitry Andric // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
22505ffd83dbSDimitry Andric // run on the SSA function just before phi elimination.
22515ffd83dbSDimitry Andric if (MO->isUndef())
22525ffd83dbSDimitry Andric report("Generic virtual register use cannot be undef", MO, MONum);
22535ffd83dbSDimitry Andric
22540eae32dcSDimitry Andric // Debug value instruction is permitted to use undefined vregs.
22550eae32dcSDimitry Andric // This is a performance measure to skip the overhead of immediately
22560eae32dcSDimitry Andric // pruning unused debug operands. The final undef substitution occurs
22570eae32dcSDimitry Andric // when debug values are allocated in LDVImpl::handleDebugValue, so
22580eae32dcSDimitry Andric // these verifications always apply after this pass.
22590eae32dcSDimitry Andric if (isFunctionTracksDebugUserValues || !MO->isUse() ||
22600eae32dcSDimitry Andric !MI->isDebugValue() || !MRI->def_empty(Reg)) {
22610b57cec5SDimitry Andric // If we're post-Select, we can't have gvregs anymore.
22620b57cec5SDimitry Andric if (isFunctionSelected) {
22630b57cec5SDimitry Andric report("Generic virtual register invalid in a Selected function",
22640b57cec5SDimitry Andric MO, MONum);
22650b57cec5SDimitry Andric return;
22660b57cec5SDimitry Andric }
22670b57cec5SDimitry Andric
22680b57cec5SDimitry Andric // The gvreg must have a type and it must not have a SubIdx.
22690b57cec5SDimitry Andric LLT Ty = MRI->getType(Reg);
22700b57cec5SDimitry Andric if (!Ty.isValid()) {
22710b57cec5SDimitry Andric report("Generic virtual register must have a valid type", MO,
22720b57cec5SDimitry Andric MONum);
22730b57cec5SDimitry Andric return;
22740b57cec5SDimitry Andric }
22750b57cec5SDimitry Andric
22760b57cec5SDimitry Andric const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
227706c3fb27SDimitry Andric const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
22780b57cec5SDimitry Andric
22790b57cec5SDimitry Andric // If we're post-RegBankSelect, the gvreg must have a bank.
22800b57cec5SDimitry Andric if (!RegBank && isFunctionRegBankSelected) {
22810b57cec5SDimitry Andric report("Generic virtual register must have a bank in a "
22820b57cec5SDimitry Andric "RegBankSelected function",
22830b57cec5SDimitry Andric MO, MONum);
22840b57cec5SDimitry Andric return;
22850b57cec5SDimitry Andric }
22860b57cec5SDimitry Andric
22870b57cec5SDimitry Andric // Make sure the register fits into its register bank if any.
22885f757f3fSDimitry Andric if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
228906c3fb27SDimitry Andric RBI->getMaximumSize(RegBank->getID()) < Ty.getSizeInBits()) {
22900b57cec5SDimitry Andric report("Register bank is too small for virtual register", MO,
22910b57cec5SDimitry Andric MONum);
22920b57cec5SDimitry Andric errs() << "Register bank " << RegBank->getName() << " too small("
229306c3fb27SDimitry Andric << RBI->getMaximumSize(RegBank->getID()) << ") to fit "
229406c3fb27SDimitry Andric << Ty.getSizeInBits() << "-bits\n";
22950b57cec5SDimitry Andric return;
22960b57cec5SDimitry Andric }
22970eae32dcSDimitry Andric }
22980eae32dcSDimitry Andric
22990b57cec5SDimitry Andric if (SubIdx) {
23000b57cec5SDimitry Andric report("Generic virtual register does not allow subregister index", MO,
23010b57cec5SDimitry Andric MONum);
23020b57cec5SDimitry Andric return;
23030b57cec5SDimitry Andric }
23040b57cec5SDimitry Andric
23050b57cec5SDimitry Andric // If this is a target specific instruction and this operand
23060b57cec5SDimitry Andric // has register class constraint, the virtual register must
23070b57cec5SDimitry Andric // comply to it.
23080b57cec5SDimitry Andric if (!isPreISelGenericOpcode(MCID.getOpcode()) &&
23090b57cec5SDimitry Andric MONum < MCID.getNumOperands() &&
23100b57cec5SDimitry Andric TII->getRegClass(MCID, MONum, TRI, *MF)) {
23110b57cec5SDimitry Andric report("Virtual register does not match instruction constraint", MO,
23120b57cec5SDimitry Andric MONum);
23130b57cec5SDimitry Andric errs() << "Expect register class "
23140b57cec5SDimitry Andric << TRI->getRegClassName(
23150b57cec5SDimitry Andric TII->getRegClass(MCID, MONum, TRI, *MF))
23160b57cec5SDimitry Andric << " but got nothing\n";
23170b57cec5SDimitry Andric return;
23180b57cec5SDimitry Andric }
23190b57cec5SDimitry Andric
23200b57cec5SDimitry Andric break;
23210b57cec5SDimitry Andric }
23220b57cec5SDimitry Andric if (SubIdx) {
23230b57cec5SDimitry Andric const TargetRegisterClass *SRC =
23240b57cec5SDimitry Andric TRI->getSubClassWithSubReg(RC, SubIdx);
23250b57cec5SDimitry Andric if (!SRC) {
23260b57cec5SDimitry Andric report("Invalid subregister index for virtual register", MO, MONum);
23270b57cec5SDimitry Andric errs() << "Register class " << TRI->getRegClassName(RC)
23280b57cec5SDimitry Andric << " does not support subreg index " << SubIdx << "\n";
23290b57cec5SDimitry Andric return;
23300b57cec5SDimitry Andric }
23310b57cec5SDimitry Andric if (RC != SRC) {
23320b57cec5SDimitry Andric report("Invalid register class for subregister index", MO, MONum);
23330b57cec5SDimitry Andric errs() << "Register class " << TRI->getRegClassName(RC)
23340b57cec5SDimitry Andric << " does not fully support subreg index " << SubIdx << "\n";
23350b57cec5SDimitry Andric return;
23360b57cec5SDimitry Andric }
23370b57cec5SDimitry Andric }
23380b57cec5SDimitry Andric if (MONum < MCID.getNumOperands()) {
23390b57cec5SDimitry Andric if (const TargetRegisterClass *DRC =
23400b57cec5SDimitry Andric TII->getRegClass(MCID, MONum, TRI, *MF)) {
23410b57cec5SDimitry Andric if (SubIdx) {
23420b57cec5SDimitry Andric const TargetRegisterClass *SuperRC =
23430b57cec5SDimitry Andric TRI->getLargestLegalSuperClass(RC, *MF);
23440b57cec5SDimitry Andric if (!SuperRC) {
23450b57cec5SDimitry Andric report("No largest legal super class exists.", MO, MONum);
23460b57cec5SDimitry Andric return;
23470b57cec5SDimitry Andric }
23480b57cec5SDimitry Andric DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx);
23490b57cec5SDimitry Andric if (!DRC) {
23500b57cec5SDimitry Andric report("No matching super-reg register class.", MO, MONum);
23510b57cec5SDimitry Andric return;
23520b57cec5SDimitry Andric }
23530b57cec5SDimitry Andric }
23540b57cec5SDimitry Andric if (!RC->hasSuperClassEq(DRC)) {
23550b57cec5SDimitry Andric report("Illegal virtual register for instruction", MO, MONum);
23560b57cec5SDimitry Andric errs() << "Expected a " << TRI->getRegClassName(DRC)
23570b57cec5SDimitry Andric << " register, but got a " << TRI->getRegClassName(RC)
23580b57cec5SDimitry Andric << " register\n";
23590b57cec5SDimitry Andric }
23600b57cec5SDimitry Andric }
23610b57cec5SDimitry Andric }
23620b57cec5SDimitry Andric }
23630b57cec5SDimitry Andric break;
23640b57cec5SDimitry Andric }
23650b57cec5SDimitry Andric
23660b57cec5SDimitry Andric case MachineOperand::MO_RegisterMask:
23670b57cec5SDimitry Andric regMasks.push_back(MO->getRegMask());
23680b57cec5SDimitry Andric break;
23690b57cec5SDimitry Andric
23700b57cec5SDimitry Andric case MachineOperand::MO_MachineBasicBlock:
23710b57cec5SDimitry Andric if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent()))
23720b57cec5SDimitry Andric report("PHI operand is not in the CFG", MO, MONum);
23730b57cec5SDimitry Andric break;
23740b57cec5SDimitry Andric
23750b57cec5SDimitry Andric case MachineOperand::MO_FrameIndex:
23760b57cec5SDimitry Andric if (LiveStks && LiveStks->hasInterval(MO->getIndex()) &&
23770b57cec5SDimitry Andric LiveInts && !LiveInts->isNotInMIMap(*MI)) {
23780b57cec5SDimitry Andric int FI = MO->getIndex();
23790b57cec5SDimitry Andric LiveInterval &LI = LiveStks->getInterval(FI);
23800b57cec5SDimitry Andric SlotIndex Idx = LiveInts->getInstructionIndex(*MI);
23810b57cec5SDimitry Andric
23820b57cec5SDimitry Andric bool stores = MI->mayStore();
23830b57cec5SDimitry Andric bool loads = MI->mayLoad();
23840b57cec5SDimitry Andric // For a memory-to-memory move, we need to check if the frame
23850b57cec5SDimitry Andric // index is used for storing or loading, by inspecting the
23860b57cec5SDimitry Andric // memory operands.
23870b57cec5SDimitry Andric if (stores && loads) {
23880b57cec5SDimitry Andric for (auto *MMO : MI->memoperands()) {
23890b57cec5SDimitry Andric const PseudoSourceValue *PSV = MMO->getPseudoValue();
23900b57cec5SDimitry Andric if (PSV == nullptr) continue;
23910b57cec5SDimitry Andric const FixedStackPseudoSourceValue *Value =
23920b57cec5SDimitry Andric dyn_cast<FixedStackPseudoSourceValue>(PSV);
23930b57cec5SDimitry Andric if (Value == nullptr) continue;
23940b57cec5SDimitry Andric if (Value->getFrameIndex() != FI) continue;
23950b57cec5SDimitry Andric
23960b57cec5SDimitry Andric if (MMO->isStore())
23970b57cec5SDimitry Andric loads = false;
23980b57cec5SDimitry Andric else
23990b57cec5SDimitry Andric stores = false;
24000b57cec5SDimitry Andric break;
24010b57cec5SDimitry Andric }
24020b57cec5SDimitry Andric if (loads == stores)
24030b57cec5SDimitry Andric report("Missing fixed stack memoperand.", MI);
24040b57cec5SDimitry Andric }
24050b57cec5SDimitry Andric if (loads && !LI.liveAt(Idx.getRegSlot(true))) {
24060b57cec5SDimitry Andric report("Instruction loads from dead spill slot", MO, MONum);
24070b57cec5SDimitry Andric errs() << "Live stack: " << LI << '\n';
24080b57cec5SDimitry Andric }
24090b57cec5SDimitry Andric if (stores && !LI.liveAt(Idx.getRegSlot())) {
24100b57cec5SDimitry Andric report("Instruction stores to dead spill slot", MO, MONum);
24110b57cec5SDimitry Andric errs() << "Live stack: " << LI << '\n';
24120b57cec5SDimitry Andric }
24130b57cec5SDimitry Andric }
24140b57cec5SDimitry Andric break;
24150b57cec5SDimitry Andric
241681ad6265SDimitry Andric case MachineOperand::MO_CFIIndex:
241781ad6265SDimitry Andric if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
241881ad6265SDimitry Andric report("CFI instruction has invalid index", MO, MONum);
241981ad6265SDimitry Andric break;
242081ad6265SDimitry Andric
24210b57cec5SDimitry Andric default:
24220b57cec5SDimitry Andric break;
24230b57cec5SDimitry Andric }
24240b57cec5SDimitry Andric }
24250b57cec5SDimitry Andric
checkLivenessAtUse(const MachineOperand * MO,unsigned MONum,SlotIndex UseIdx,const LiveRange & LR,Register VRegOrUnit,LaneBitmask LaneMask)24260b57cec5SDimitry Andric void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2427e8d8bef9SDimitry Andric unsigned MONum, SlotIndex UseIdx,
2428e8d8bef9SDimitry Andric const LiveRange &LR,
2429e8d8bef9SDimitry Andric Register VRegOrUnit,
24300b57cec5SDimitry Andric LaneBitmask LaneMask) {
24315f757f3fSDimitry Andric const MachineInstr *MI = MO->getParent();
24320b57cec5SDimitry Andric LiveQueryResult LRQ = LR.Query(UseIdx);
24335f757f3fSDimitry Andric bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
24340b57cec5SDimitry Andric // Check if we have a segment at the use, note however that we only need one
24350b57cec5SDimitry Andric // live subregister range, the others may be dead.
24365f757f3fSDimitry Andric if (!HasValue && LaneMask.none()) {
24370b57cec5SDimitry Andric report("No live segment at use", MO, MONum);
24380b57cec5SDimitry Andric report_context_liverange(LR);
24390b57cec5SDimitry Andric report_context_vreg_regunit(VRegOrUnit);
24400b57cec5SDimitry Andric report_context(UseIdx);
24410b57cec5SDimitry Andric }
24420b57cec5SDimitry Andric if (MO->isKill() && !LRQ.isKill()) {
24430b57cec5SDimitry Andric report("Live range continues after kill flag", MO, MONum);
24440b57cec5SDimitry Andric report_context_liverange(LR);
24450b57cec5SDimitry Andric report_context_vreg_regunit(VRegOrUnit);
24460b57cec5SDimitry Andric if (LaneMask.any())
24470b57cec5SDimitry Andric report_context_lanemask(LaneMask);
24480b57cec5SDimitry Andric report_context(UseIdx);
24490b57cec5SDimitry Andric }
24500b57cec5SDimitry Andric }
24510b57cec5SDimitry Andric
checkLivenessAtDef(const MachineOperand * MO,unsigned MONum,SlotIndex DefIdx,const LiveRange & LR,Register VRegOrUnit,bool SubRangeCheck,LaneBitmask LaneMask)24520b57cec5SDimitry Andric void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
2453e8d8bef9SDimitry Andric unsigned MONum, SlotIndex DefIdx,
2454e8d8bef9SDimitry Andric const LiveRange &LR,
2455e8d8bef9SDimitry Andric Register VRegOrUnit,
2456e8d8bef9SDimitry Andric bool SubRangeCheck,
2457e8d8bef9SDimitry Andric LaneBitmask LaneMask) {
24580b57cec5SDimitry Andric if (const VNInfo *VNI = LR.getVNInfoAt(DefIdx)) {
2459bdd1243dSDimitry Andric // The LR can correspond to the whole reg and its def slot is not obliged
2460bdd1243dSDimitry Andric // to be the same as the MO' def slot. E.g. when we check here "normal"
2461bdd1243dSDimitry Andric // subreg MO but there is other EC subreg MO in the same instruction so the
2462bdd1243dSDimitry Andric // whole reg has EC def slot and differs from the currently checked MO' def
2463bdd1243dSDimitry Andric // slot. For example:
2464bdd1243dSDimitry Andric // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
2465bdd1243dSDimitry Andric // Check that there is an early-clobber def of the same superregister
2466bdd1243dSDimitry Andric // somewhere is performed in visitMachineFunctionAfter()
2467bdd1243dSDimitry Andric if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
2468bdd1243dSDimitry Andric !SlotIndex::isSameInstr(VNI->def, DefIdx) ||
2469bdd1243dSDimitry Andric (VNI->def != DefIdx &&
2470bdd1243dSDimitry Andric (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
24710b57cec5SDimitry Andric report("Inconsistent valno->def", MO, MONum);
24720b57cec5SDimitry Andric report_context_liverange(LR);
24730b57cec5SDimitry Andric report_context_vreg_regunit(VRegOrUnit);
24740b57cec5SDimitry Andric if (LaneMask.any())
24750b57cec5SDimitry Andric report_context_lanemask(LaneMask);
24760b57cec5SDimitry Andric report_context(*VNI);
24770b57cec5SDimitry Andric report_context(DefIdx);
24780b57cec5SDimitry Andric }
24790b57cec5SDimitry Andric } else {
24800b57cec5SDimitry Andric report("No live segment at def", MO, MONum);
24810b57cec5SDimitry Andric report_context_liverange(LR);
24820b57cec5SDimitry Andric report_context_vreg_regunit(VRegOrUnit);
24830b57cec5SDimitry Andric if (LaneMask.any())
24840b57cec5SDimitry Andric report_context_lanemask(LaneMask);
24850b57cec5SDimitry Andric report_context(DefIdx);
24860b57cec5SDimitry Andric }
24870b57cec5SDimitry Andric // Check that, if the dead def flag is present, LiveInts agree.
24880b57cec5SDimitry Andric if (MO->isDead()) {
24890b57cec5SDimitry Andric LiveQueryResult LRQ = LR.Query(DefIdx);
24900b57cec5SDimitry Andric if (!LRQ.isDeadDef()) {
2491bdd1243dSDimitry Andric assert(VRegOrUnit.isVirtual() && "Expecting a virtual register.");
24920b57cec5SDimitry Andric // A dead subreg def only tells us that the specific subreg is dead. There
24930b57cec5SDimitry Andric // could be other non-dead defs of other subregs, or we could have other
24940b57cec5SDimitry Andric // parts of the register being live through the instruction. So unless we
24950b57cec5SDimitry Andric // are checking liveness for a subrange it is ok for the live range to
24960b57cec5SDimitry Andric // continue, given that we have a dead def of a subregister.
24970b57cec5SDimitry Andric if (SubRangeCheck || MO->getSubReg() == 0) {
24980b57cec5SDimitry Andric report("Live range continues after dead def flag", MO, MONum);
24990b57cec5SDimitry Andric report_context_liverange(LR);
25000b57cec5SDimitry Andric report_context_vreg_regunit(VRegOrUnit);
25010b57cec5SDimitry Andric if (LaneMask.any())
25020b57cec5SDimitry Andric report_context_lanemask(LaneMask);
25030b57cec5SDimitry Andric }
25040b57cec5SDimitry Andric }
25050b57cec5SDimitry Andric }
25060b57cec5SDimitry Andric }
25070b57cec5SDimitry Andric
checkLiveness(const MachineOperand * MO,unsigned MONum)25080b57cec5SDimitry Andric void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
25090b57cec5SDimitry Andric const MachineInstr *MI = MO->getParent();
2510e8d8bef9SDimitry Andric const Register Reg = MO->getReg();
2511349cc55cSDimitry Andric const unsigned SubRegIdx = MO->getSubReg();
2512349cc55cSDimitry Andric
2513349cc55cSDimitry Andric const LiveInterval *LI = nullptr;
2514349cc55cSDimitry Andric if (LiveInts && Reg.isVirtual()) {
2515349cc55cSDimitry Andric if (LiveInts->hasInterval(Reg)) {
2516349cc55cSDimitry Andric LI = &LiveInts->getInterval(Reg);
25170eae32dcSDimitry Andric if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
25180eae32dcSDimitry Andric !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(Reg))
2519349cc55cSDimitry Andric report("Live interval for subreg operand has no subranges", MO, MONum);
2520349cc55cSDimitry Andric } else {
2521349cc55cSDimitry Andric report("Virtual register has no live interval", MO, MONum);
2522349cc55cSDimitry Andric }
2523349cc55cSDimitry Andric }
25240b57cec5SDimitry Andric
25250b57cec5SDimitry Andric // Both use and def operands can read a register.
25260b57cec5SDimitry Andric if (MO->readsReg()) {
25270b57cec5SDimitry Andric if (MO->isKill())
25280b57cec5SDimitry Andric addRegWithSubRegs(regsKilled, Reg);
25290b57cec5SDimitry Andric
2530349cc55cSDimitry Andric // Check that LiveVars knows this kill (unless we are inside a bundle, in
2531349cc55cSDimitry Andric // which case we have already checked that LiveVars knows any kills on the
2532349cc55cSDimitry Andric // bundle header instead).
2533349cc55cSDimitry Andric if (LiveVars && Reg.isVirtual() && MO->isKill() &&
2534349cc55cSDimitry Andric !MI->isBundledWithPred()) {
25350b57cec5SDimitry Andric LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
25360b57cec5SDimitry Andric if (!is_contained(VI.Kills, MI))
25370b57cec5SDimitry Andric report("Kill missing from LiveVariables", MO, MONum);
25380b57cec5SDimitry Andric }
25390b57cec5SDimitry Andric
25400b57cec5SDimitry Andric // Check LiveInts liveness and kill.
25410b57cec5SDimitry Andric if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {
25425f757f3fSDimitry Andric SlotIndex UseIdx;
25435f757f3fSDimitry Andric if (MI->isPHI()) {
25445f757f3fSDimitry Andric // PHI use occurs on the edge, so check for live out here instead.
25455f757f3fSDimitry Andric UseIdx = LiveInts->getMBBEndIdx(
25465f757f3fSDimitry Andric MI->getOperand(MONum + 1).getMBB()).getPrevSlot();
25475f757f3fSDimitry Andric } else {
25485f757f3fSDimitry Andric UseIdx = LiveInts->getInstructionIndex(*MI);
25495f757f3fSDimitry Andric }
25500b57cec5SDimitry Andric // Check the cached regunit intervals.
2551e8d8bef9SDimitry Andric if (Reg.isPhysical() && !isReserved(Reg)) {
255206c3fb27SDimitry Andric for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg())) {
255306c3fb27SDimitry Andric if (MRI->isReservedRegUnit(Unit))
25540b57cec5SDimitry Andric continue;
255506c3fb27SDimitry Andric if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
255606c3fb27SDimitry Andric checkLivenessAtUse(MO, MONum, UseIdx, *LR, Unit);
25570b57cec5SDimitry Andric }
25580b57cec5SDimitry Andric }
25590b57cec5SDimitry Andric
2560349cc55cSDimitry Andric if (Reg.isVirtual()) {
25610b57cec5SDimitry Andric // This is a virtual register interval.
2562349cc55cSDimitry Andric checkLivenessAtUse(MO, MONum, UseIdx, *LI, Reg);
25630b57cec5SDimitry Andric
2564349cc55cSDimitry Andric if (LI->hasSubRanges() && !MO->isDef()) {
25650b57cec5SDimitry Andric LaneBitmask MOMask = SubRegIdx != 0
25660b57cec5SDimitry Andric ? TRI->getSubRegIndexLaneMask(SubRegIdx)
25670b57cec5SDimitry Andric : MRI->getMaxLaneMaskForVReg(Reg);
25680b57cec5SDimitry Andric LaneBitmask LiveInMask;
2569349cc55cSDimitry Andric for (const LiveInterval::SubRange &SR : LI->subranges()) {
25700b57cec5SDimitry Andric if ((MOMask & SR.LaneMask).none())
25710b57cec5SDimitry Andric continue;
25720b57cec5SDimitry Andric checkLivenessAtUse(MO, MONum, UseIdx, SR, Reg, SR.LaneMask);
25730b57cec5SDimitry Andric LiveQueryResult LRQ = SR.Query(UseIdx);
25745f757f3fSDimitry Andric if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
25750b57cec5SDimitry Andric LiveInMask |= SR.LaneMask;
25760b57cec5SDimitry Andric }
25770b57cec5SDimitry Andric // At least parts of the register has to be live at the use.
25780b57cec5SDimitry Andric if ((LiveInMask & MOMask).none()) {
25790b57cec5SDimitry Andric report("No live subrange at use", MO, MONum);
2580349cc55cSDimitry Andric report_context(*LI);
25810b57cec5SDimitry Andric report_context(UseIdx);
25820b57cec5SDimitry Andric }
25835f757f3fSDimitry Andric // For PHIs all lanes should be live
25845f757f3fSDimitry Andric if (MI->isPHI() && LiveInMask != MOMask) {
25855f757f3fSDimitry Andric report("Not all lanes of PHI source live at use", MO, MONum);
25865f757f3fSDimitry Andric report_context(*LI);
25875f757f3fSDimitry Andric report_context(UseIdx);
25885f757f3fSDimitry Andric }
25890b57cec5SDimitry Andric }
25900b57cec5SDimitry Andric }
25910b57cec5SDimitry Andric }
25920b57cec5SDimitry Andric
25930b57cec5SDimitry Andric // Use of a dead register.
25940b57cec5SDimitry Andric if (!regsLive.count(Reg)) {
2595349cc55cSDimitry Andric if (Reg.isPhysical()) {
25960b57cec5SDimitry Andric // Reserved registers may be used even when 'dead'.
25970b57cec5SDimitry Andric bool Bad = !isReserved(Reg);
25980b57cec5SDimitry Andric // We are fine if just any subregister has a defined value.
25990b57cec5SDimitry Andric if (Bad) {
2600480093f4SDimitry Andric
2601480093f4SDimitry Andric for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
2602480093f4SDimitry Andric if (regsLive.count(SubReg)) {
26030b57cec5SDimitry Andric Bad = false;
26040b57cec5SDimitry Andric break;
26050b57cec5SDimitry Andric }
26060b57cec5SDimitry Andric }
26070b57cec5SDimitry Andric }
26080b57cec5SDimitry Andric // If there is an additional implicit-use of a super register we stop
26090b57cec5SDimitry Andric // here. By definition we are fine if the super register is not
26100b57cec5SDimitry Andric // (completely) dead, if the complete super register is dead we will
26110b57cec5SDimitry Andric // get a report for its operand.
26120b57cec5SDimitry Andric if (Bad) {
26130b57cec5SDimitry Andric for (const MachineOperand &MOP : MI->uses()) {
26140b57cec5SDimitry Andric if (!MOP.isReg() || !MOP.isImplicit())
26150b57cec5SDimitry Andric continue;
26160b57cec5SDimitry Andric
2617349cc55cSDimitry Andric if (!MOP.getReg().isPhysical())
26180b57cec5SDimitry Andric continue;
26190b57cec5SDimitry Andric
2620fe6060f1SDimitry Andric if (llvm::is_contained(TRI->subregs(MOP.getReg()), Reg))
26210b57cec5SDimitry Andric Bad = false;
26220b57cec5SDimitry Andric }
26230b57cec5SDimitry Andric }
26240b57cec5SDimitry Andric if (Bad)
26250b57cec5SDimitry Andric report("Using an undefined physical register", MO, MONum);
26260b57cec5SDimitry Andric } else if (MRI->def_empty(Reg)) {
26270b57cec5SDimitry Andric report("Reading virtual register without a def", MO, MONum);
26280b57cec5SDimitry Andric } else {
26290b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[MI->getParent()];
26300b57cec5SDimitry Andric // We don't know which virtual registers are live in, so only complain
26310b57cec5SDimitry Andric // if vreg was killed in this MBB. Otherwise keep track of vregs that
26320b57cec5SDimitry Andric // must be live in. PHI instructions are handled separately.
26330b57cec5SDimitry Andric if (MInfo.regsKilled.count(Reg))
26340b57cec5SDimitry Andric report("Using a killed virtual register", MO, MONum);
26350b57cec5SDimitry Andric else if (!MI->isPHI())
26360b57cec5SDimitry Andric MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI));
26370b57cec5SDimitry Andric }
26380b57cec5SDimitry Andric }
26390b57cec5SDimitry Andric }
26400b57cec5SDimitry Andric
26410b57cec5SDimitry Andric if (MO->isDef()) {
26420b57cec5SDimitry Andric // Register defined.
26430b57cec5SDimitry Andric // TODO: verify that earlyclobber ops are not used.
26440b57cec5SDimitry Andric if (MO->isDead())
26450b57cec5SDimitry Andric addRegWithSubRegs(regsDead, Reg);
26460b57cec5SDimitry Andric else
26470b57cec5SDimitry Andric addRegWithSubRegs(regsDefined, Reg);
26480b57cec5SDimitry Andric
26490b57cec5SDimitry Andric // Verify SSA form.
2650349cc55cSDimitry Andric if (MRI->isSSA() && Reg.isVirtual() &&
26510b57cec5SDimitry Andric std::next(MRI->def_begin(Reg)) != MRI->def_end())
26520b57cec5SDimitry Andric report("Multiple virtual register defs in SSA form", MO, MONum);
26530b57cec5SDimitry Andric
26540b57cec5SDimitry Andric // Check LiveInts for a live segment, but only for virtual registers.
26550b57cec5SDimitry Andric if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {
26560b57cec5SDimitry Andric SlotIndex DefIdx = LiveInts->getInstructionIndex(*MI);
26570b57cec5SDimitry Andric DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber());
26580b57cec5SDimitry Andric
2659349cc55cSDimitry Andric if (Reg.isVirtual()) {
2660349cc55cSDimitry Andric checkLivenessAtDef(MO, MONum, DefIdx, *LI, Reg);
26610b57cec5SDimitry Andric
2662349cc55cSDimitry Andric if (LI->hasSubRanges()) {
26630b57cec5SDimitry Andric LaneBitmask MOMask = SubRegIdx != 0
26640b57cec5SDimitry Andric ? TRI->getSubRegIndexLaneMask(SubRegIdx)
26650b57cec5SDimitry Andric : MRI->getMaxLaneMaskForVReg(Reg);
2666349cc55cSDimitry Andric for (const LiveInterval::SubRange &SR : LI->subranges()) {
26670b57cec5SDimitry Andric if ((SR.LaneMask & MOMask).none())
26680b57cec5SDimitry Andric continue;
26690b57cec5SDimitry Andric checkLivenessAtDef(MO, MONum, DefIdx, SR, Reg, true, SR.LaneMask);
26700b57cec5SDimitry Andric }
26710b57cec5SDimitry Andric }
26720b57cec5SDimitry Andric }
26730b57cec5SDimitry Andric }
26740b57cec5SDimitry Andric }
26750b57cec5SDimitry Andric }
26760b57cec5SDimitry Andric
26770b57cec5SDimitry Andric // This function gets called after visiting all instructions in a bundle. The
26780b57cec5SDimitry Andric // argument points to the bundle header.
26790b57cec5SDimitry Andric // Normal stand-alone instructions are also considered 'bundles', and this
26800b57cec5SDimitry Andric // function is called for all of them.
visitMachineBundleAfter(const MachineInstr * MI)26810b57cec5SDimitry Andric void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
26820b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[MI->getParent()];
26830b57cec5SDimitry Andric set_union(MInfo.regsKilled, regsKilled);
26840b57cec5SDimitry Andric set_subtract(regsLive, regsKilled); regsKilled.clear();
26850b57cec5SDimitry Andric // Kill any masked registers.
26860b57cec5SDimitry Andric while (!regMasks.empty()) {
26870b57cec5SDimitry Andric const uint32_t *Mask = regMasks.pop_back_val();
2688e8d8bef9SDimitry Andric for (Register Reg : regsLive)
2689e8d8bef9SDimitry Andric if (Reg.isPhysical() &&
2690e8d8bef9SDimitry Andric MachineOperand::clobbersPhysReg(Mask, Reg.asMCReg()))
26915ffd83dbSDimitry Andric regsDead.push_back(Reg);
26920b57cec5SDimitry Andric }
26930b57cec5SDimitry Andric set_subtract(regsLive, regsDead); regsDead.clear();
26940b57cec5SDimitry Andric set_union(regsLive, regsDefined); regsDefined.clear();
26950b57cec5SDimitry Andric }
26960b57cec5SDimitry Andric
26970b57cec5SDimitry Andric void
visitMachineBasicBlockAfter(const MachineBasicBlock * MBB)26980b57cec5SDimitry Andric MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
26990b57cec5SDimitry Andric MBBInfoMap[MBB].regsLiveOut = regsLive;
27000b57cec5SDimitry Andric regsLive.clear();
27010b57cec5SDimitry Andric
27020b57cec5SDimitry Andric if (Indexes) {
27030b57cec5SDimitry Andric SlotIndex stop = Indexes->getMBBEndIdx(MBB);
27040b57cec5SDimitry Andric if (!(stop > lastIndex)) {
27050b57cec5SDimitry Andric report("Block ends before last instruction index", MBB);
27060b57cec5SDimitry Andric errs() << "Block ends at " << stop
27070b57cec5SDimitry Andric << " last instruction was at " << lastIndex << '\n';
27080b57cec5SDimitry Andric }
27090b57cec5SDimitry Andric lastIndex = stop;
27100b57cec5SDimitry Andric }
27110b57cec5SDimitry Andric }
27120b57cec5SDimitry Andric
27135ffd83dbSDimitry Andric namespace {
27145ffd83dbSDimitry Andric // This implements a set of registers that serves as a filter: can filter other
27155ffd83dbSDimitry Andric // sets by passing through elements not in the filter and blocking those that
27165ffd83dbSDimitry Andric // are. Any filter implicitly includes the full set of physical registers upon
27175ffd83dbSDimitry Andric // creation, thus filtering them all out. The filter itself as a set only grows,
27185ffd83dbSDimitry Andric // and needs to be as efficient as possible.
27195ffd83dbSDimitry Andric struct VRegFilter {
27205ffd83dbSDimitry Andric // Add elements to the filter itself. \pre Input set \p FromRegSet must have
27215ffd83dbSDimitry Andric // no duplicates. Both virtual and physical registers are fine.
add__anoncaf807cd0511::VRegFilter27225ffd83dbSDimitry Andric template <typename RegSetT> void add(const RegSetT &FromRegSet) {
2723e8d8bef9SDimitry Andric SmallVector<Register, 0> VRegsBuffer;
27245ffd83dbSDimitry Andric filterAndAdd(FromRegSet, VRegsBuffer);
27255ffd83dbSDimitry Andric }
27265ffd83dbSDimitry Andric // Filter \p FromRegSet through the filter and append passed elements into \p
27275ffd83dbSDimitry Andric // ToVRegs. All elements appended are then added to the filter itself.
27285ffd83dbSDimitry Andric // \returns true if anything changed.
27295ffd83dbSDimitry Andric template <typename RegSetT>
filterAndAdd__anoncaf807cd0511::VRegFilter27305ffd83dbSDimitry Andric bool filterAndAdd(const RegSetT &FromRegSet,
2731e8d8bef9SDimitry Andric SmallVectorImpl<Register> &ToVRegs) {
27325ffd83dbSDimitry Andric unsigned SparseUniverse = Sparse.size();
27335ffd83dbSDimitry Andric unsigned NewSparseUniverse = SparseUniverse;
27345ffd83dbSDimitry Andric unsigned NewDenseSize = Dense.size();
27355ffd83dbSDimitry Andric size_t Begin = ToVRegs.size();
2736e8d8bef9SDimitry Andric for (Register Reg : FromRegSet) {
2737e8d8bef9SDimitry Andric if (!Reg.isVirtual())
27385ffd83dbSDimitry Andric continue;
27395ffd83dbSDimitry Andric unsigned Index = Register::virtReg2Index(Reg);
27405ffd83dbSDimitry Andric if (Index < SparseUniverseMax) {
27415ffd83dbSDimitry Andric if (Index < SparseUniverse && Sparse.test(Index))
27425ffd83dbSDimitry Andric continue;
27435ffd83dbSDimitry Andric NewSparseUniverse = std::max(NewSparseUniverse, Index + 1);
27445ffd83dbSDimitry Andric } else {
27455ffd83dbSDimitry Andric if (Dense.count(Reg))
27465ffd83dbSDimitry Andric continue;
27475ffd83dbSDimitry Andric ++NewDenseSize;
27485ffd83dbSDimitry Andric }
27495ffd83dbSDimitry Andric ToVRegs.push_back(Reg);
27505ffd83dbSDimitry Andric }
27515ffd83dbSDimitry Andric size_t End = ToVRegs.size();
27525ffd83dbSDimitry Andric if (Begin == End)
27535ffd83dbSDimitry Andric return false;
27545ffd83dbSDimitry Andric // Reserving space in sets once performs better than doing so continuously
27555ffd83dbSDimitry Andric // and pays easily for double look-ups (even in Dense with SparseUniverseMax
27565ffd83dbSDimitry Andric // tuned all the way down) and double iteration (the second one is over a
27575ffd83dbSDimitry Andric // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
27585ffd83dbSDimitry Andric Sparse.resize(NewSparseUniverse);
27595ffd83dbSDimitry Andric Dense.reserve(NewDenseSize);
27605ffd83dbSDimitry Andric for (unsigned I = Begin; I < End; ++I) {
2761e8d8bef9SDimitry Andric Register Reg = ToVRegs[I];
27625ffd83dbSDimitry Andric unsigned Index = Register::virtReg2Index(Reg);
27635ffd83dbSDimitry Andric if (Index < SparseUniverseMax)
27645ffd83dbSDimitry Andric Sparse.set(Index);
27655ffd83dbSDimitry Andric else
27665ffd83dbSDimitry Andric Dense.insert(Reg);
27675ffd83dbSDimitry Andric }
27685ffd83dbSDimitry Andric return true;
27695ffd83dbSDimitry Andric }
27705ffd83dbSDimitry Andric
27715ffd83dbSDimitry Andric private:
27725ffd83dbSDimitry Andric static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
27735ffd83dbSDimitry Andric // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyound
27745ffd83dbSDimitry Andric // are tracked by Dense. The only purpose of the threashold and the Dense set
27755ffd83dbSDimitry Andric // is to have a reasonably growing memory usage in pathological cases (large
27765ffd83dbSDimitry Andric // number of very sparse VRegFilter instances live at the same time). In
27775ffd83dbSDimitry Andric // practice even in the worst-by-execution time cases having all elements
27785ffd83dbSDimitry Andric // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
27795ffd83dbSDimitry Andric // space efficient than if tracked by Dense. The threashold is set to keep the
27805ffd83dbSDimitry Andric // worst-case memory usage within 2x of figures determined empirically for
27815ffd83dbSDimitry Andric // "all Dense" scenario in such worst-by-execution-time cases.
27825ffd83dbSDimitry Andric BitVector Sparse;
27835ffd83dbSDimitry Andric DenseSet<unsigned> Dense;
27845ffd83dbSDimitry Andric };
27855ffd83dbSDimitry Andric
27865ffd83dbSDimitry Andric // Implements both a transfer function and a (binary, in-place) join operator
27875ffd83dbSDimitry Andric // for a dataflow over register sets with set union join and filtering transfer
27885ffd83dbSDimitry Andric // (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
27895ffd83dbSDimitry Andric // Maintains out_b as its state, allowing for O(n) iteration over it at any
27905ffd83dbSDimitry Andric // time, where n is the size of the set (as opposed to O(U) where U is the
27915ffd83dbSDimitry Andric // universe). filter_b implicitly contains all physical registers at all times.
27925ffd83dbSDimitry Andric class FilteringVRegSet {
27935ffd83dbSDimitry Andric VRegFilter Filter;
2794e8d8bef9SDimitry Andric SmallVector<Register, 0> VRegs;
27955ffd83dbSDimitry Andric
27965ffd83dbSDimitry Andric public:
27975ffd83dbSDimitry Andric // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
27985ffd83dbSDimitry Andric // Both virtual and physical registers are fine.
addToFilter(const RegSetT & RS)27995ffd83dbSDimitry Andric template <typename RegSetT> void addToFilter(const RegSetT &RS) {
28005ffd83dbSDimitry Andric Filter.add(RS);
28015ffd83dbSDimitry Andric }
28025ffd83dbSDimitry Andric // Passes \p RS through the filter_b (transfer function) and adds what's left
28035ffd83dbSDimitry Andric // to itself (out_b).
add(const RegSetT & RS)28045ffd83dbSDimitry Andric template <typename RegSetT> bool add(const RegSetT &RS) {
28055ffd83dbSDimitry Andric // Double-duty the Filter: to maintain VRegs a set (and the join operation
28065ffd83dbSDimitry Andric // a set union) just add everything being added here to the Filter as well.
28075ffd83dbSDimitry Andric return Filter.filterAndAdd(RS, VRegs);
28085ffd83dbSDimitry Andric }
28095ffd83dbSDimitry Andric using const_iterator = decltype(VRegs)::const_iterator;
begin() const28105ffd83dbSDimitry Andric const_iterator begin() const { return VRegs.begin(); }
end() const28115ffd83dbSDimitry Andric const_iterator end() const { return VRegs.end(); }
size() const28125ffd83dbSDimitry Andric size_t size() const { return VRegs.size(); }
28135ffd83dbSDimitry Andric };
28145ffd83dbSDimitry Andric } // namespace
28155ffd83dbSDimitry Andric
28160b57cec5SDimitry Andric // Calculate the largest possible vregsPassed sets. These are the registers that
28170b57cec5SDimitry Andric // can pass through an MBB live, but may not be live every time. It is assumed
28180b57cec5SDimitry Andric // that all vregsPassed sets are empty before the call.
calcRegsPassed()28190b57cec5SDimitry Andric void MachineVerifier::calcRegsPassed() {
2820e8d8bef9SDimitry Andric if (MF->empty())
28215ffd83dbSDimitry Andric // ReversePostOrderTraversal doesn't handle empty functions.
28225ffd83dbSDimitry Andric return;
28230b57cec5SDimitry Andric
2824e8d8bef9SDimitry Andric for (const MachineBasicBlock *MB :
2825e8d8bef9SDimitry Andric ReversePostOrderTraversal<const MachineFunction *>(MF)) {
2826e8d8bef9SDimitry Andric FilteringVRegSet VRegs;
2827e8d8bef9SDimitry Andric BBInfo &Info = MBBInfoMap[MB];
2828e8d8bef9SDimitry Andric assert(Info.reachable);
2829e8d8bef9SDimitry Andric
2830e8d8bef9SDimitry Andric VRegs.addToFilter(Info.regsKilled);
2831e8d8bef9SDimitry Andric VRegs.addToFilter(Info.regsLiveOut);
2832e8d8bef9SDimitry Andric for (const MachineBasicBlock *Pred : MB->predecessors()) {
2833e8d8bef9SDimitry Andric const BBInfo &PredInfo = MBBInfoMap[Pred];
2834e8d8bef9SDimitry Andric if (!PredInfo.reachable)
28350b57cec5SDimitry Andric continue;
2836e8d8bef9SDimitry Andric
2837e8d8bef9SDimitry Andric VRegs.add(PredInfo.regsLiveOut);
2838e8d8bef9SDimitry Andric VRegs.add(PredInfo.vregsPassed);
28390b57cec5SDimitry Andric }
2840e8d8bef9SDimitry Andric Info.vregsPassed.reserve(VRegs.size());
2841e8d8bef9SDimitry Andric Info.vregsPassed.insert(VRegs.begin(), VRegs.end());
28425ffd83dbSDimitry Andric }
28430b57cec5SDimitry Andric }
28440b57cec5SDimitry Andric
28450b57cec5SDimitry Andric // Calculate the set of virtual registers that must be passed through each basic
28460b57cec5SDimitry Andric // block in order to satisfy the requirements of successor blocks. This is very
28470b57cec5SDimitry Andric // similar to calcRegsPassed, only backwards.
calcRegsRequired()28480b57cec5SDimitry Andric void MachineVerifier::calcRegsRequired() {
28490b57cec5SDimitry Andric // First push live-in regs to predecessors' vregsRequired.
28500b57cec5SDimitry Andric SmallPtrSet<const MachineBasicBlock*, 8> todo;
28510b57cec5SDimitry Andric for (const auto &MBB : *MF) {
28520b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MBB];
28535ffd83dbSDimitry Andric for (const MachineBasicBlock *Pred : MBB.predecessors()) {
28545ffd83dbSDimitry Andric BBInfo &PInfo = MBBInfoMap[Pred];
28550b57cec5SDimitry Andric if (PInfo.addRequired(MInfo.vregsLiveIn))
28565ffd83dbSDimitry Andric todo.insert(Pred);
28570b57cec5SDimitry Andric }
2858e8d8bef9SDimitry Andric
2859e8d8bef9SDimitry Andric // Handle the PHI node.
2860e8d8bef9SDimitry Andric for (const MachineInstr &MI : MBB.phis()) {
2861e8d8bef9SDimitry Andric for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
2862e8d8bef9SDimitry Andric // Skip those Operands which are undef regs or not regs.
2863e8d8bef9SDimitry Andric if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
2864e8d8bef9SDimitry Andric continue;
2865e8d8bef9SDimitry Andric
2866e8d8bef9SDimitry Andric // Get register and predecessor for one PHI edge.
2867e8d8bef9SDimitry Andric Register Reg = MI.getOperand(i).getReg();
2868e8d8bef9SDimitry Andric const MachineBasicBlock *Pred = MI.getOperand(i + 1).getMBB();
2869e8d8bef9SDimitry Andric
2870e8d8bef9SDimitry Andric BBInfo &PInfo = MBBInfoMap[Pred];
2871e8d8bef9SDimitry Andric if (PInfo.addRequired(Reg))
2872e8d8bef9SDimitry Andric todo.insert(Pred);
2873e8d8bef9SDimitry Andric }
2874e8d8bef9SDimitry Andric }
28750b57cec5SDimitry Andric }
28760b57cec5SDimitry Andric
28770b57cec5SDimitry Andric // Iteratively push vregsRequired to predecessors. This will converge to the
28780b57cec5SDimitry Andric // same final state regardless of DenseSet iteration order.
28790b57cec5SDimitry Andric while (!todo.empty()) {
28800b57cec5SDimitry Andric const MachineBasicBlock *MBB = *todo.begin();
28810b57cec5SDimitry Andric todo.erase(MBB);
28820b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[MBB];
28835ffd83dbSDimitry Andric for (const MachineBasicBlock *Pred : MBB->predecessors()) {
28845ffd83dbSDimitry Andric if (Pred == MBB)
28850b57cec5SDimitry Andric continue;
28865ffd83dbSDimitry Andric BBInfo &SInfo = MBBInfoMap[Pred];
28870b57cec5SDimitry Andric if (SInfo.addRequired(MInfo.vregsRequired))
28885ffd83dbSDimitry Andric todo.insert(Pred);
28890b57cec5SDimitry Andric }
28900b57cec5SDimitry Andric }
28910b57cec5SDimitry Andric }
28920b57cec5SDimitry Andric
28930b57cec5SDimitry Andric // Check PHI instructions at the beginning of MBB. It is assumed that
28940b57cec5SDimitry Andric // calcRegsPassed has been run so BBInfo::isLiveOut is valid.
checkPHIOps(const MachineBasicBlock & MBB)28950b57cec5SDimitry Andric void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
28960b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MBB];
28970b57cec5SDimitry Andric
28980b57cec5SDimitry Andric SmallPtrSet<const MachineBasicBlock*, 8> seen;
28990b57cec5SDimitry Andric for (const MachineInstr &Phi : MBB) {
29000b57cec5SDimitry Andric if (!Phi.isPHI())
29010b57cec5SDimitry Andric break;
29020b57cec5SDimitry Andric seen.clear();
29030b57cec5SDimitry Andric
29040b57cec5SDimitry Andric const MachineOperand &MODef = Phi.getOperand(0);
29050b57cec5SDimitry Andric if (!MODef.isReg() || !MODef.isDef()) {
29060b57cec5SDimitry Andric report("Expected first PHI operand to be a register def", &MODef, 0);
29070b57cec5SDimitry Andric continue;
29080b57cec5SDimitry Andric }
29090b57cec5SDimitry Andric if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
29100b57cec5SDimitry Andric MODef.isEarlyClobber() || MODef.isDebug())
29110b57cec5SDimitry Andric report("Unexpected flag on PHI operand", &MODef, 0);
29128bcb0991SDimitry Andric Register DefReg = MODef.getReg();
2913bdd1243dSDimitry Andric if (!DefReg.isVirtual())
29140b57cec5SDimitry Andric report("Expected first PHI operand to be a virtual register", &MODef, 0);
29150b57cec5SDimitry Andric
29160b57cec5SDimitry Andric for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
29170b57cec5SDimitry Andric const MachineOperand &MO0 = Phi.getOperand(I);
29180b57cec5SDimitry Andric if (!MO0.isReg()) {
29190b57cec5SDimitry Andric report("Expected PHI operand to be a register", &MO0, I);
29200b57cec5SDimitry Andric continue;
29210b57cec5SDimitry Andric }
29220b57cec5SDimitry Andric if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
29230b57cec5SDimitry Andric MO0.isDebug() || MO0.isTied())
29240b57cec5SDimitry Andric report("Unexpected flag on PHI operand", &MO0, I);
29250b57cec5SDimitry Andric
29260b57cec5SDimitry Andric const MachineOperand &MO1 = Phi.getOperand(I + 1);
29270b57cec5SDimitry Andric if (!MO1.isMBB()) {
29280b57cec5SDimitry Andric report("Expected PHI operand to be a basic block", &MO1, I + 1);
29290b57cec5SDimitry Andric continue;
29300b57cec5SDimitry Andric }
29310b57cec5SDimitry Andric
29320b57cec5SDimitry Andric const MachineBasicBlock &Pre = *MO1.getMBB();
29330b57cec5SDimitry Andric if (!Pre.isSuccessor(&MBB)) {
29340b57cec5SDimitry Andric report("PHI input is not a predecessor block", &MO1, I + 1);
29350b57cec5SDimitry Andric continue;
29360b57cec5SDimitry Andric }
29370b57cec5SDimitry Andric
29380b57cec5SDimitry Andric if (MInfo.reachable) {
29390b57cec5SDimitry Andric seen.insert(&Pre);
29400b57cec5SDimitry Andric BBInfo &PrInfo = MBBInfoMap[&Pre];
29410b57cec5SDimitry Andric if (!MO0.isUndef() && PrInfo.reachable &&
29420b57cec5SDimitry Andric !PrInfo.isLiveOut(MO0.getReg()))
29430b57cec5SDimitry Andric report("PHI operand is not live-out from predecessor", &MO0, I);
29440b57cec5SDimitry Andric }
29450b57cec5SDimitry Andric }
29460b57cec5SDimitry Andric
29470b57cec5SDimitry Andric // Did we see all predecessors?
29480b57cec5SDimitry Andric if (MInfo.reachable) {
29490b57cec5SDimitry Andric for (MachineBasicBlock *Pred : MBB.predecessors()) {
29500b57cec5SDimitry Andric if (!seen.count(Pred)) {
29510b57cec5SDimitry Andric report("Missing PHI operand", &Phi);
29520b57cec5SDimitry Andric errs() << printMBBReference(*Pred)
29530b57cec5SDimitry Andric << " is a predecessor according to the CFG.\n";
29540b57cec5SDimitry Andric }
29550b57cec5SDimitry Andric }
29560b57cec5SDimitry Andric }
29570b57cec5SDimitry Andric }
29580b57cec5SDimitry Andric }
29590b57cec5SDimitry Andric
visitMachineFunctionAfter()29600b57cec5SDimitry Andric void MachineVerifier::visitMachineFunctionAfter() {
29610b57cec5SDimitry Andric calcRegsPassed();
29620b57cec5SDimitry Andric
29630b57cec5SDimitry Andric for (const MachineBasicBlock &MBB : *MF)
29640b57cec5SDimitry Andric checkPHIOps(MBB);
29650b57cec5SDimitry Andric
29660b57cec5SDimitry Andric // Now check liveness info if available
29670b57cec5SDimitry Andric calcRegsRequired();
29680b57cec5SDimitry Andric
29690b57cec5SDimitry Andric // Check for killed virtual registers that should be live out.
29700b57cec5SDimitry Andric for (const auto &MBB : *MF) {
29710b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MBB];
2972e8d8bef9SDimitry Andric for (Register VReg : MInfo.vregsRequired)
29735ffd83dbSDimitry Andric if (MInfo.regsKilled.count(VReg)) {
29740b57cec5SDimitry Andric report("Virtual register killed in block, but needed live out.", &MBB);
29755ffd83dbSDimitry Andric errs() << "Virtual register " << printReg(VReg)
29760b57cec5SDimitry Andric << " is used after the block.\n";
29770b57cec5SDimitry Andric }
29780b57cec5SDimitry Andric }
29790b57cec5SDimitry Andric
29800b57cec5SDimitry Andric if (!MF->empty()) {
29810b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MF->front()];
2982e8d8bef9SDimitry Andric for (Register VReg : MInfo.vregsRequired) {
29830b57cec5SDimitry Andric report("Virtual register defs don't dominate all uses.", MF);
29845ffd83dbSDimitry Andric report_context_vreg(VReg);
29850b57cec5SDimitry Andric }
29860b57cec5SDimitry Andric }
29870b57cec5SDimitry Andric
29880b57cec5SDimitry Andric if (LiveVars)
29890b57cec5SDimitry Andric verifyLiveVariables();
29900b57cec5SDimitry Andric if (LiveInts)
29910b57cec5SDimitry Andric verifyLiveIntervals();
29920b57cec5SDimitry Andric
2993480093f4SDimitry Andric // Check live-in list of each MBB. If a register is live into MBB, check
2994480093f4SDimitry Andric // that the register is in regsLiveOut of each predecessor block. Since
2995480093f4SDimitry Andric // this must come from a definition in the predecesssor or its live-in
2996480093f4SDimitry Andric // list, this will catch a live-through case where the predecessor does not
2997480093f4SDimitry Andric // have the register in its live-in list. This currently only checks
2998480093f4SDimitry Andric // registers that have no aliases, are not allocatable and are not
2999480093f4SDimitry Andric // reserved, which could mean a condition code register for instance.
3000480093f4SDimitry Andric if (MRI->tracksLiveness())
3001480093f4SDimitry Andric for (const auto &MBB : *MF)
3002480093f4SDimitry Andric for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3003480093f4SDimitry Andric MCPhysReg LiveInReg = P.PhysReg;
3004480093f4SDimitry Andric bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3005480093f4SDimitry Andric if (hasAliases || isAllocatable(LiveInReg) || isReserved(LiveInReg))
3006480093f4SDimitry Andric continue;
3007480093f4SDimitry Andric for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3008480093f4SDimitry Andric BBInfo &PInfo = MBBInfoMap[Pred];
3009480093f4SDimitry Andric if (!PInfo.regsLiveOut.count(LiveInReg)) {
3010480093f4SDimitry Andric report("Live in register not found to be live out from predecessor.",
3011480093f4SDimitry Andric &MBB);
3012480093f4SDimitry Andric errs() << TRI->getName(LiveInReg)
3013480093f4SDimitry Andric << " not found to be live out from "
3014480093f4SDimitry Andric << printMBBReference(*Pred) << "\n";
3015480093f4SDimitry Andric }
3016480093f4SDimitry Andric }
3017480093f4SDimitry Andric }
3018480093f4SDimitry Andric
30190b57cec5SDimitry Andric for (auto CSInfo : MF->getCallSitesInfo())
30200b57cec5SDimitry Andric if (!CSInfo.first->isCall())
30210b57cec5SDimitry Andric report("Call site info referencing instruction that is not call", MF);
3022e8d8bef9SDimitry Andric
3023e8d8bef9SDimitry Andric // If there's debug-info, check that we don't have any duplicate value
3024e8d8bef9SDimitry Andric // tracking numbers.
3025e8d8bef9SDimitry Andric if (MF->getFunction().getSubprogram()) {
3026e8d8bef9SDimitry Andric DenseSet<unsigned> SeenNumbers;
3027fcaf7f86SDimitry Andric for (const auto &MBB : *MF) {
3028fcaf7f86SDimitry Andric for (const auto &MI : MBB) {
3029e8d8bef9SDimitry Andric if (auto Num = MI.peekDebugInstrNum()) {
3030e8d8bef9SDimitry Andric auto Result = SeenNumbers.insert((unsigned)Num);
3031e8d8bef9SDimitry Andric if (!Result.second)
3032e8d8bef9SDimitry Andric report("Instruction has a duplicated value tracking number", &MI);
3033e8d8bef9SDimitry Andric }
3034e8d8bef9SDimitry Andric }
3035e8d8bef9SDimitry Andric }
3036e8d8bef9SDimitry Andric }
30370b57cec5SDimitry Andric }
30380b57cec5SDimitry Andric
verifyLiveVariables()30390b57cec5SDimitry Andric void MachineVerifier::verifyLiveVariables() {
30400b57cec5SDimitry Andric assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");
3041e8d8bef9SDimitry Andric for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3042e8d8bef9SDimitry Andric Register Reg = Register::index2VirtReg(I);
30430b57cec5SDimitry Andric LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
30440b57cec5SDimitry Andric for (const auto &MBB : *MF) {
30450b57cec5SDimitry Andric BBInfo &MInfo = MBBInfoMap[&MBB];
30460b57cec5SDimitry Andric
30470b57cec5SDimitry Andric // Our vregsRequired should be identical to LiveVariables' AliveBlocks
30480b57cec5SDimitry Andric if (MInfo.vregsRequired.count(Reg)) {
30490b57cec5SDimitry Andric if (!VI.AliveBlocks.test(MBB.getNumber())) {
30500b57cec5SDimitry Andric report("LiveVariables: Block missing from AliveBlocks", &MBB);
30510b57cec5SDimitry Andric errs() << "Virtual register " << printReg(Reg)
30520b57cec5SDimitry Andric << " must be live through the block.\n";
30530b57cec5SDimitry Andric }
30540b57cec5SDimitry Andric } else {
30550b57cec5SDimitry Andric if (VI.AliveBlocks.test(MBB.getNumber())) {
30560b57cec5SDimitry Andric report("LiveVariables: Block should not be in AliveBlocks", &MBB);
30570b57cec5SDimitry Andric errs() << "Virtual register " << printReg(Reg)
30580b57cec5SDimitry Andric << " is not needed live through the block.\n";
30590b57cec5SDimitry Andric }
30600b57cec5SDimitry Andric }
30610b57cec5SDimitry Andric }
30620b57cec5SDimitry Andric }
30630b57cec5SDimitry Andric }
30640b57cec5SDimitry Andric
verifyLiveIntervals()30650b57cec5SDimitry Andric void MachineVerifier::verifyLiveIntervals() {
30660b57cec5SDimitry Andric assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3067e8d8bef9SDimitry Andric for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3068e8d8bef9SDimitry Andric Register Reg = Register::index2VirtReg(I);
30690b57cec5SDimitry Andric
30700b57cec5SDimitry Andric // Spilling and splitting may leave unused registers around. Skip them.
30710b57cec5SDimitry Andric if (MRI->reg_nodbg_empty(Reg))
30720b57cec5SDimitry Andric continue;
30730b57cec5SDimitry Andric
30740b57cec5SDimitry Andric if (!LiveInts->hasInterval(Reg)) {
30750b57cec5SDimitry Andric report("Missing live interval for virtual register", MF);
30760b57cec5SDimitry Andric errs() << printReg(Reg, TRI) << " still has defs or uses\n";
30770b57cec5SDimitry Andric continue;
30780b57cec5SDimitry Andric }
30790b57cec5SDimitry Andric
30800b57cec5SDimitry Andric const LiveInterval &LI = LiveInts->getInterval(Reg);
3081e8d8bef9SDimitry Andric assert(Reg == LI.reg() && "Invalid reg to interval mapping");
30820b57cec5SDimitry Andric verifyLiveInterval(LI);
30830b57cec5SDimitry Andric }
30840b57cec5SDimitry Andric
30850b57cec5SDimitry Andric // Verify all the cached regunit intervals.
30860b57cec5SDimitry Andric for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i)
30870b57cec5SDimitry Andric if (const LiveRange *LR = LiveInts->getCachedRegUnit(i))
30880b57cec5SDimitry Andric verifyLiveRange(*LR, i);
30890b57cec5SDimitry Andric }
30900b57cec5SDimitry Andric
verifyLiveRangeValue(const LiveRange & LR,const VNInfo * VNI,Register Reg,LaneBitmask LaneMask)30910b57cec5SDimitry Andric void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3092e8d8bef9SDimitry Andric const VNInfo *VNI, Register Reg,
30930b57cec5SDimitry Andric LaneBitmask LaneMask) {
30940b57cec5SDimitry Andric if (VNI->isUnused())
30950b57cec5SDimitry Andric return;
30960b57cec5SDimitry Andric
30970b57cec5SDimitry Andric const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def);
30980b57cec5SDimitry Andric
30990b57cec5SDimitry Andric if (!DefVNI) {
31000b57cec5SDimitry Andric report("Value not live at VNInfo def and not marked unused", MF);
31010b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31020b57cec5SDimitry Andric report_context(*VNI);
31030b57cec5SDimitry Andric return;
31040b57cec5SDimitry Andric }
31050b57cec5SDimitry Andric
31060b57cec5SDimitry Andric if (DefVNI != VNI) {
31070b57cec5SDimitry Andric report("Live segment at def has different VNInfo", MF);
31080b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31090b57cec5SDimitry Andric report_context(*VNI);
31100b57cec5SDimitry Andric return;
31110b57cec5SDimitry Andric }
31120b57cec5SDimitry Andric
31130b57cec5SDimitry Andric const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);
31140b57cec5SDimitry Andric if (!MBB) {
31150b57cec5SDimitry Andric report("Invalid VNInfo definition index", MF);
31160b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31170b57cec5SDimitry Andric report_context(*VNI);
31180b57cec5SDimitry Andric return;
31190b57cec5SDimitry Andric }
31200b57cec5SDimitry Andric
31210b57cec5SDimitry Andric if (VNI->isPHIDef()) {
31220b57cec5SDimitry Andric if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {
31230b57cec5SDimitry Andric report("PHIDef VNInfo is not defined at MBB start", MBB);
31240b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31250b57cec5SDimitry Andric report_context(*VNI);
31260b57cec5SDimitry Andric }
31270b57cec5SDimitry Andric return;
31280b57cec5SDimitry Andric }
31290b57cec5SDimitry Andric
31300b57cec5SDimitry Andric // Non-PHI def.
31310b57cec5SDimitry Andric const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);
31320b57cec5SDimitry Andric if (!MI) {
31330b57cec5SDimitry Andric report("No instruction at VNInfo def index", MBB);
31340b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31350b57cec5SDimitry Andric report_context(*VNI);
31360b57cec5SDimitry Andric return;
31370b57cec5SDimitry Andric }
31380b57cec5SDimitry Andric
31390b57cec5SDimitry Andric if (Reg != 0) {
31400b57cec5SDimitry Andric bool hasDef = false;
31410b57cec5SDimitry Andric bool isEarlyClobber = false;
31420b57cec5SDimitry Andric for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
31430b57cec5SDimitry Andric if (!MOI->isReg() || !MOI->isDef())
31440b57cec5SDimitry Andric continue;
3145bdd1243dSDimitry Andric if (Reg.isVirtual()) {
31460b57cec5SDimitry Andric if (MOI->getReg() != Reg)
31470b57cec5SDimitry Andric continue;
31480b57cec5SDimitry Andric } else {
3149bdd1243dSDimitry Andric if (!MOI->getReg().isPhysical() || !TRI->hasRegUnit(MOI->getReg(), Reg))
31500b57cec5SDimitry Andric continue;
31510b57cec5SDimitry Andric }
31520b57cec5SDimitry Andric if (LaneMask.any() &&
31530b57cec5SDimitry Andric (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
31540b57cec5SDimitry Andric continue;
31550b57cec5SDimitry Andric hasDef = true;
31560b57cec5SDimitry Andric if (MOI->isEarlyClobber())
31570b57cec5SDimitry Andric isEarlyClobber = true;
31580b57cec5SDimitry Andric }
31590b57cec5SDimitry Andric
31600b57cec5SDimitry Andric if (!hasDef) {
31610b57cec5SDimitry Andric report("Defining instruction does not modify register", MI);
31620b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31630b57cec5SDimitry Andric report_context(*VNI);
31640b57cec5SDimitry Andric }
31650b57cec5SDimitry Andric
31660b57cec5SDimitry Andric // Early clobber defs begin at USE slots, but other defs must begin at
31670b57cec5SDimitry Andric // DEF slots.
31680b57cec5SDimitry Andric if (isEarlyClobber) {
31690b57cec5SDimitry Andric if (!VNI->def.isEarlyClobber()) {
31700b57cec5SDimitry Andric report("Early clobber def must be at an early-clobber slot", MBB);
31710b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31720b57cec5SDimitry Andric report_context(*VNI);
31730b57cec5SDimitry Andric }
31740b57cec5SDimitry Andric } else if (!VNI->def.isRegister()) {
31750b57cec5SDimitry Andric report("Non-PHI, non-early clobber def must be at a register slot", MBB);
31760b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31770b57cec5SDimitry Andric report_context(*VNI);
31780b57cec5SDimitry Andric }
31790b57cec5SDimitry Andric }
31800b57cec5SDimitry Andric }
31810b57cec5SDimitry Andric
verifyLiveRangeSegment(const LiveRange & LR,const LiveRange::const_iterator I,Register Reg,LaneBitmask LaneMask)31820b57cec5SDimitry Andric void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
31830b57cec5SDimitry Andric const LiveRange::const_iterator I,
3184e8d8bef9SDimitry Andric Register Reg,
3185e8d8bef9SDimitry Andric LaneBitmask LaneMask) {
31860b57cec5SDimitry Andric const LiveRange::Segment &S = *I;
31870b57cec5SDimitry Andric const VNInfo *VNI = S.valno;
31880b57cec5SDimitry Andric assert(VNI && "Live segment has no valno");
31890b57cec5SDimitry Andric
31900b57cec5SDimitry Andric if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) {
31910b57cec5SDimitry Andric report("Foreign valno in live segment", MF);
31920b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
31930b57cec5SDimitry Andric report_context(S);
31940b57cec5SDimitry Andric report_context(*VNI);
31950b57cec5SDimitry Andric }
31960b57cec5SDimitry Andric
31970b57cec5SDimitry Andric if (VNI->isUnused()) {
31980b57cec5SDimitry Andric report("Live segment valno is marked unused", MF);
31990b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32000b57cec5SDimitry Andric report_context(S);
32010b57cec5SDimitry Andric }
32020b57cec5SDimitry Andric
32030b57cec5SDimitry Andric const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start);
32040b57cec5SDimitry Andric if (!MBB) {
32050b57cec5SDimitry Andric report("Bad start of live segment, no basic block", MF);
32060b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32070b57cec5SDimitry Andric report_context(S);
32080b57cec5SDimitry Andric return;
32090b57cec5SDimitry Andric }
32100b57cec5SDimitry Andric SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);
32110b57cec5SDimitry Andric if (S.start != MBBStartIdx && S.start != VNI->def) {
32120b57cec5SDimitry Andric report("Live segment must begin at MBB entry or valno def", MBB);
32130b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32140b57cec5SDimitry Andric report_context(S);
32150b57cec5SDimitry Andric }
32160b57cec5SDimitry Andric
32170b57cec5SDimitry Andric const MachineBasicBlock *EndMBB =
32180b57cec5SDimitry Andric LiveInts->getMBBFromIndex(S.end.getPrevSlot());
32190b57cec5SDimitry Andric if (!EndMBB) {
32200b57cec5SDimitry Andric report("Bad end of live segment, no basic block", MF);
32210b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32220b57cec5SDimitry Andric report_context(S);
32230b57cec5SDimitry Andric return;
32240b57cec5SDimitry Andric }
32250b57cec5SDimitry Andric
322606c3fb27SDimitry Andric // Checks for non-live-out segments.
322706c3fb27SDimitry Andric if (S.end != LiveInts->getMBBEndIdx(EndMBB)) {
32280b57cec5SDimitry Andric // RegUnit intervals are allowed dead phis.
3229bdd1243dSDimitry Andric if (!Reg.isVirtual() && VNI->isPHIDef() && S.start == VNI->def &&
3230bdd1243dSDimitry Andric S.end == VNI->def.getDeadSlot())
32310b57cec5SDimitry Andric return;
32320b57cec5SDimitry Andric
32330b57cec5SDimitry Andric // The live segment is ending inside EndMBB
32340b57cec5SDimitry Andric const MachineInstr *MI =
32350b57cec5SDimitry Andric LiveInts->getInstructionFromIndex(S.end.getPrevSlot());
32360b57cec5SDimitry Andric if (!MI) {
32370b57cec5SDimitry Andric report("Live segment doesn't end at a valid instruction", EndMBB);
32380b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32390b57cec5SDimitry Andric report_context(S);
32400b57cec5SDimitry Andric return;
32410b57cec5SDimitry Andric }
32420b57cec5SDimitry Andric
32430b57cec5SDimitry Andric // The block slot must refer to a basic block boundary.
32440b57cec5SDimitry Andric if (S.end.isBlock()) {
32450b57cec5SDimitry Andric report("Live segment ends at B slot of an instruction", EndMBB);
32460b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32470b57cec5SDimitry Andric report_context(S);
32480b57cec5SDimitry Andric }
32490b57cec5SDimitry Andric
32500b57cec5SDimitry Andric if (S.end.isDead()) {
32510b57cec5SDimitry Andric // Segment ends on the dead slot.
32520b57cec5SDimitry Andric // That means there must be a dead def.
32530b57cec5SDimitry Andric if (!SlotIndex::isSameInstr(S.start, S.end)) {
32540b57cec5SDimitry Andric report("Live segment ending at dead slot spans instructions", EndMBB);
32550b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32560b57cec5SDimitry Andric report_context(S);
32570b57cec5SDimitry Andric }
32580b57cec5SDimitry Andric }
32590b57cec5SDimitry Andric
3260349cc55cSDimitry Andric // After tied operands are rewritten, a live segment can only end at an
3261349cc55cSDimitry Andric // early-clobber slot if it is being redefined by an early-clobber def.
326206c3fb27SDimitry Andric // TODO: Before tied operands are rewritten, a live segment can only end at
326306c3fb27SDimitry Andric // an early-clobber slot if the last use is tied to an early-clobber def.
3264349cc55cSDimitry Andric if (MF->getProperties().hasProperty(
3265349cc55cSDimitry Andric MachineFunctionProperties::Property::TiedOpsRewritten) &&
3266349cc55cSDimitry Andric S.end.isEarlyClobber()) {
32670b57cec5SDimitry Andric if (I + 1 == LR.end() || (I + 1)->start != S.end) {
32680b57cec5SDimitry Andric report("Live segment ending at early clobber slot must be "
326906c3fb27SDimitry Andric "redefined by an EC def in the same instruction",
327006c3fb27SDimitry Andric EndMBB);
32710b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
32720b57cec5SDimitry Andric report_context(S);
32730b57cec5SDimitry Andric }
32740b57cec5SDimitry Andric }
32750b57cec5SDimitry Andric
32760b57cec5SDimitry Andric // The following checks only apply to virtual registers. Physreg liveness
32770b57cec5SDimitry Andric // is too weird to check.
3278bdd1243dSDimitry Andric if (Reg.isVirtual()) {
32790b57cec5SDimitry Andric // A live segment can end with either a redefinition, a kill flag on a
32800b57cec5SDimitry Andric // use, or a dead flag on a def.
32810b57cec5SDimitry Andric bool hasRead = false;
32820b57cec5SDimitry Andric bool hasSubRegDef = false;
32830b57cec5SDimitry Andric bool hasDeadDef = false;
32840b57cec5SDimitry Andric for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
32850b57cec5SDimitry Andric if (!MOI->isReg() || MOI->getReg() != Reg)
32860b57cec5SDimitry Andric continue;
32870b57cec5SDimitry Andric unsigned Sub = MOI->getSubReg();
328806c3fb27SDimitry Andric LaneBitmask SLM =
328906c3fb27SDimitry Andric Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : LaneBitmask::getAll();
32900b57cec5SDimitry Andric if (MOI->isDef()) {
32910b57cec5SDimitry Andric if (Sub != 0) {
32920b57cec5SDimitry Andric hasSubRegDef = true;
32930b57cec5SDimitry Andric // An operand %0:sub0 reads %0:sub1..n. Invert the lane
32940b57cec5SDimitry Andric // mask for subregister defs. Read-undef defs will be handled by
32950b57cec5SDimitry Andric // readsReg below.
32960b57cec5SDimitry Andric SLM = ~SLM;
32970b57cec5SDimitry Andric }
32980b57cec5SDimitry Andric if (MOI->isDead())
32990b57cec5SDimitry Andric hasDeadDef = true;
33000b57cec5SDimitry Andric }
33010b57cec5SDimitry Andric if (LaneMask.any() && (LaneMask & SLM).none())
33020b57cec5SDimitry Andric continue;
33030b57cec5SDimitry Andric if (MOI->readsReg())
33040b57cec5SDimitry Andric hasRead = true;
33050b57cec5SDimitry Andric }
33060b57cec5SDimitry Andric if (S.end.isDead()) {
33070b57cec5SDimitry Andric // Make sure that the corresponding machine operand for a "dead" live
33080b57cec5SDimitry Andric // range has the dead flag. We cannot perform this check for subregister
33090b57cec5SDimitry Andric // liveranges as partially dead values are allowed.
33100b57cec5SDimitry Andric if (LaneMask.none() && !hasDeadDef) {
331106c3fb27SDimitry Andric report(
331206c3fb27SDimitry Andric "Instruction ending live segment on dead slot has no dead flag",
33130b57cec5SDimitry Andric MI);
33140b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
33150b57cec5SDimitry Andric report_context(S);
33160b57cec5SDimitry Andric }
33170b57cec5SDimitry Andric } else {
33180b57cec5SDimitry Andric if (!hasRead) {
33190b57cec5SDimitry Andric // When tracking subregister liveness, the main range must start new
33200b57cec5SDimitry Andric // values on partial register writes, even if there is no read.
33210b57cec5SDimitry Andric if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask.any() ||
33220b57cec5SDimitry Andric !hasSubRegDef) {
33230b57cec5SDimitry Andric report("Instruction ending live segment doesn't read the register",
33240b57cec5SDimitry Andric MI);
33250b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
33260b57cec5SDimitry Andric report_context(S);
33270b57cec5SDimitry Andric }
33280b57cec5SDimitry Andric }
33290b57cec5SDimitry Andric }
33300b57cec5SDimitry Andric }
333106c3fb27SDimitry Andric }
33320b57cec5SDimitry Andric
33330b57cec5SDimitry Andric // Now check all the basic blocks in this live segment.
33340b57cec5SDimitry Andric MachineFunction::const_iterator MFI = MBB->getIterator();
33350b57cec5SDimitry Andric // Is this live segment the beginning of a non-PHIDef VN?
33360b57cec5SDimitry Andric if (S.start == VNI->def && !VNI->isPHIDef()) {
33370b57cec5SDimitry Andric // Not live-in to any blocks.
33380b57cec5SDimitry Andric if (MBB == EndMBB)
33390b57cec5SDimitry Andric return;
33400b57cec5SDimitry Andric // Skip this block.
33410b57cec5SDimitry Andric ++MFI;
33420b57cec5SDimitry Andric }
33430b57cec5SDimitry Andric
33440b57cec5SDimitry Andric SmallVector<SlotIndex, 4> Undefs;
33450b57cec5SDimitry Andric if (LaneMask.any()) {
33460b57cec5SDimitry Andric LiveInterval &OwnerLI = LiveInts->getInterval(Reg);
33470b57cec5SDimitry Andric OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, *MRI, *Indexes);
33480b57cec5SDimitry Andric }
33490b57cec5SDimitry Andric
33500b57cec5SDimitry Andric while (true) {
33510b57cec5SDimitry Andric assert(LiveInts->isLiveInToMBB(LR, &*MFI));
33520b57cec5SDimitry Andric // We don't know how to track physregs into a landing pad.
3353bdd1243dSDimitry Andric if (!Reg.isVirtual() && MFI->isEHPad()) {
33540b57cec5SDimitry Andric if (&*MFI == EndMBB)
33550b57cec5SDimitry Andric break;
33560b57cec5SDimitry Andric ++MFI;
33570b57cec5SDimitry Andric continue;
33580b57cec5SDimitry Andric }
33590b57cec5SDimitry Andric
33600b57cec5SDimitry Andric // Is VNI a PHI-def in the current block?
33610b57cec5SDimitry Andric bool IsPHI = VNI->isPHIDef() &&
33620b57cec5SDimitry Andric VNI->def == LiveInts->getMBBStartIdx(&*MFI);
33630b57cec5SDimitry Andric
33640b57cec5SDimitry Andric // Check that VNI is live-out of all predecessors.
33655ffd83dbSDimitry Andric for (const MachineBasicBlock *Pred : MFI->predecessors()) {
33665ffd83dbSDimitry Andric SlotIndex PEnd = LiveInts->getMBBEndIdx(Pred);
3367fe6060f1SDimitry Andric // Predecessor of landing pad live-out on last call.
3368fe6060f1SDimitry Andric if (MFI->isEHPad()) {
33694824e7fdSDimitry Andric for (const MachineInstr &MI : llvm::reverse(*Pred)) {
33704824e7fdSDimitry Andric if (MI.isCall()) {
33714824e7fdSDimitry Andric PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3372fe6060f1SDimitry Andric break;
3373fe6060f1SDimitry Andric }
3374fe6060f1SDimitry Andric }
3375fe6060f1SDimitry Andric }
33760b57cec5SDimitry Andric const VNInfo *PVNI = LR.getVNInfoBefore(PEnd);
33770b57cec5SDimitry Andric
33780b57cec5SDimitry Andric // All predecessors must have a live-out value. However for a phi
33790b57cec5SDimitry Andric // instruction with subregister intervals
33800b57cec5SDimitry Andric // only one of the subregisters (not necessarily the current one) needs to
33810b57cec5SDimitry Andric // be defined.
33820b57cec5SDimitry Andric if (!PVNI && (LaneMask.none() || !IsPHI)) {
33835ffd83dbSDimitry Andric if (LiveRangeCalc::isJointlyDominated(Pred, Undefs, *Indexes))
33840b57cec5SDimitry Andric continue;
33855ffd83dbSDimitry Andric report("Register not marked live out of predecessor", Pred);
33860b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
33870b57cec5SDimitry Andric report_context(*VNI);
33880b57cec5SDimitry Andric errs() << " live into " << printMBBReference(*MFI) << '@'
33890b57cec5SDimitry Andric << LiveInts->getMBBStartIdx(&*MFI) << ", not live before "
33900b57cec5SDimitry Andric << PEnd << '\n';
33910b57cec5SDimitry Andric continue;
33920b57cec5SDimitry Andric }
33930b57cec5SDimitry Andric
33940b57cec5SDimitry Andric // Only PHI-defs can take different predecessor values.
33950b57cec5SDimitry Andric if (!IsPHI && PVNI != VNI) {
33965ffd83dbSDimitry Andric report("Different value live out of predecessor", Pred);
33970b57cec5SDimitry Andric report_context(LR, Reg, LaneMask);
33980b57cec5SDimitry Andric errs() << "Valno #" << PVNI->id << " live out of "
33995ffd83dbSDimitry Andric << printMBBReference(*Pred) << '@' << PEnd << "\nValno #"
34000b57cec5SDimitry Andric << VNI->id << " live into " << printMBBReference(*MFI) << '@'
34010b57cec5SDimitry Andric << LiveInts->getMBBStartIdx(&*MFI) << '\n';
34020b57cec5SDimitry Andric }
34030b57cec5SDimitry Andric }
34040b57cec5SDimitry Andric if (&*MFI == EndMBB)
34050b57cec5SDimitry Andric break;
34060b57cec5SDimitry Andric ++MFI;
34070b57cec5SDimitry Andric }
34080b57cec5SDimitry Andric }
34090b57cec5SDimitry Andric
verifyLiveRange(const LiveRange & LR,Register Reg,LaneBitmask LaneMask)3410e8d8bef9SDimitry Andric void MachineVerifier::verifyLiveRange(const LiveRange &LR, Register Reg,
34110b57cec5SDimitry Andric LaneBitmask LaneMask) {
34120b57cec5SDimitry Andric for (const VNInfo *VNI : LR.valnos)
34130b57cec5SDimitry Andric verifyLiveRangeValue(LR, VNI, Reg, LaneMask);
34140b57cec5SDimitry Andric
34150b57cec5SDimitry Andric for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
34160b57cec5SDimitry Andric verifyLiveRangeSegment(LR, I, Reg, LaneMask);
34170b57cec5SDimitry Andric }
34180b57cec5SDimitry Andric
verifyLiveInterval(const LiveInterval & LI)34190b57cec5SDimitry Andric void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3420e8d8bef9SDimitry Andric Register Reg = LI.reg();
3421bdd1243dSDimitry Andric assert(Reg.isVirtual());
34220b57cec5SDimitry Andric verifyLiveRange(LI, Reg);
34230b57cec5SDimitry Andric
34245f757f3fSDimitry Andric if (LI.hasSubRanges()) {
34250b57cec5SDimitry Andric LaneBitmask Mask;
34260b57cec5SDimitry Andric LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
34270b57cec5SDimitry Andric for (const LiveInterval::SubRange &SR : LI.subranges()) {
34280b57cec5SDimitry Andric if ((Mask & SR.LaneMask).any()) {
34290b57cec5SDimitry Andric report("Lane masks of sub ranges overlap in live interval", MF);
34300b57cec5SDimitry Andric report_context(LI);
34310b57cec5SDimitry Andric }
34320b57cec5SDimitry Andric if ((SR.LaneMask & ~MaxMask).any()) {
34330b57cec5SDimitry Andric report("Subrange lanemask is invalid", MF);
34340b57cec5SDimitry Andric report_context(LI);
34350b57cec5SDimitry Andric }
34360b57cec5SDimitry Andric if (SR.empty()) {
34370b57cec5SDimitry Andric report("Subrange must not be empty", MF);
3438e8d8bef9SDimitry Andric report_context(SR, LI.reg(), SR.LaneMask);
34390b57cec5SDimitry Andric }
34400b57cec5SDimitry Andric Mask |= SR.LaneMask;
3441e8d8bef9SDimitry Andric verifyLiveRange(SR, LI.reg(), SR.LaneMask);
34420b57cec5SDimitry Andric if (!LI.covers(SR)) {
34430b57cec5SDimitry Andric report("A Subrange is not covered by the main range", MF);
34440b57cec5SDimitry Andric report_context(LI);
34450b57cec5SDimitry Andric }
34460b57cec5SDimitry Andric }
34475f757f3fSDimitry Andric }
34480b57cec5SDimitry Andric
34490b57cec5SDimitry Andric // Check the LI only has one connected component.
34500b57cec5SDimitry Andric ConnectedVNInfoEqClasses ConEQ(*LiveInts);
34510b57cec5SDimitry Andric unsigned NumComp = ConEQ.Classify(LI);
34520b57cec5SDimitry Andric if (NumComp > 1) {
34530b57cec5SDimitry Andric report("Multiple connected components in live interval", MF);
34540b57cec5SDimitry Andric report_context(LI);
34550b57cec5SDimitry Andric for (unsigned comp = 0; comp != NumComp; ++comp) {
34560b57cec5SDimitry Andric errs() << comp << ": valnos";
34575ffd83dbSDimitry Andric for (const VNInfo *I : LI.valnos)
34585ffd83dbSDimitry Andric if (comp == ConEQ.getEqClass(I))
34595ffd83dbSDimitry Andric errs() << ' ' << I->id;
34600b57cec5SDimitry Andric errs() << '\n';
34610b57cec5SDimitry Andric }
34620b57cec5SDimitry Andric }
34630b57cec5SDimitry Andric }
34640b57cec5SDimitry Andric
34650b57cec5SDimitry Andric namespace {
34660b57cec5SDimitry Andric
34670b57cec5SDimitry Andric // FrameSetup and FrameDestroy can have zero adjustment, so using a single
34680b57cec5SDimitry Andric // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
34690b57cec5SDimitry Andric // value is zero.
34700b57cec5SDimitry Andric // We use a bool plus an integer to capture the stack state.
34710b57cec5SDimitry Andric struct StackStateOfBB {
34720b57cec5SDimitry Andric StackStateOfBB() = default;
StackStateOfBB__anoncaf807cd0611::StackStateOfBB34730b57cec5SDimitry Andric StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup) :
34740b57cec5SDimitry Andric EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
34750b57cec5SDimitry Andric ExitIsSetup(ExitSetup) {}
34760b57cec5SDimitry Andric
34770b57cec5SDimitry Andric // Can be negative, which means we are setting up a frame.
34780b57cec5SDimitry Andric int EntryValue = 0;
34790b57cec5SDimitry Andric int ExitValue = 0;
34800b57cec5SDimitry Andric bool EntryIsSetup = false;
34810b57cec5SDimitry Andric bool ExitIsSetup = false;
34820b57cec5SDimitry Andric };
34830b57cec5SDimitry Andric
34840b57cec5SDimitry Andric } // end anonymous namespace
34850b57cec5SDimitry Andric
34860b57cec5SDimitry Andric /// Make sure on every path through the CFG, a FrameSetup <n> is always followed
34870b57cec5SDimitry Andric /// by a FrameDestroy <n>, stack adjustments are identical on all
34880b57cec5SDimitry Andric /// CFG edges to a merge point, and frame is destroyed at end of a return block.
verifyStackFrame()34890b57cec5SDimitry Andric void MachineVerifier::verifyStackFrame() {
34900b57cec5SDimitry Andric unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
34910b57cec5SDimitry Andric unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
34920b57cec5SDimitry Andric if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
34930b57cec5SDimitry Andric return;
34940b57cec5SDimitry Andric
34950b57cec5SDimitry Andric SmallVector<StackStateOfBB, 8> SPState;
34960b57cec5SDimitry Andric SPState.resize(MF->getNumBlockIDs());
34970b57cec5SDimitry Andric df_iterator_default_set<const MachineBasicBlock*> Reachable;
34980b57cec5SDimitry Andric
34990b57cec5SDimitry Andric // Visit the MBBs in DFS order.
35000b57cec5SDimitry Andric for (df_ext_iterator<const MachineFunction *,
35010b57cec5SDimitry Andric df_iterator_default_set<const MachineBasicBlock *>>
35020b57cec5SDimitry Andric DFI = df_ext_begin(MF, Reachable), DFE = df_ext_end(MF, Reachable);
35030b57cec5SDimitry Andric DFI != DFE; ++DFI) {
35040b57cec5SDimitry Andric const MachineBasicBlock *MBB = *DFI;
35050b57cec5SDimitry Andric
35060b57cec5SDimitry Andric StackStateOfBB BBState;
35070b57cec5SDimitry Andric // Check the exit state of the DFS stack predecessor.
35080b57cec5SDimitry Andric if (DFI.getPathLength() >= 2) {
35090b57cec5SDimitry Andric const MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2);
35100b57cec5SDimitry Andric assert(Reachable.count(StackPred) &&
35110b57cec5SDimitry Andric "DFS stack predecessor is already visited.\n");
35120b57cec5SDimitry Andric BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
35130b57cec5SDimitry Andric BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
35140b57cec5SDimitry Andric BBState.ExitValue = BBState.EntryValue;
35150b57cec5SDimitry Andric BBState.ExitIsSetup = BBState.EntryIsSetup;
35160b57cec5SDimitry Andric }
35170b57cec5SDimitry Andric
35185f757f3fSDimitry Andric if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
35195f757f3fSDimitry Andric report("Call frame size on entry does not match value computed from "
35205f757f3fSDimitry Andric "predecessor",
35215f757f3fSDimitry Andric MBB);
35225f757f3fSDimitry Andric errs() << "Call frame size on entry " << MBB->getCallFrameSize()
35235f757f3fSDimitry Andric << " does not match value computed from predecessor "
35245f757f3fSDimitry Andric << -BBState.EntryValue << '\n';
35255f757f3fSDimitry Andric }
35265f757f3fSDimitry Andric
35270b57cec5SDimitry Andric // Update stack state by checking contents of MBB.
35280b57cec5SDimitry Andric for (const auto &I : *MBB) {
35290b57cec5SDimitry Andric if (I.getOpcode() == FrameSetupOpcode) {
35300b57cec5SDimitry Andric if (BBState.ExitIsSetup)
35310b57cec5SDimitry Andric report("FrameSetup is after another FrameSetup", &I);
35320b57cec5SDimitry Andric BBState.ExitValue -= TII->getFrameTotalSize(I);
35330b57cec5SDimitry Andric BBState.ExitIsSetup = true;
35340b57cec5SDimitry Andric }
35350b57cec5SDimitry Andric
35360b57cec5SDimitry Andric if (I.getOpcode() == FrameDestroyOpcode) {
35370b57cec5SDimitry Andric int Size = TII->getFrameTotalSize(I);
35380b57cec5SDimitry Andric if (!BBState.ExitIsSetup)
35390b57cec5SDimitry Andric report("FrameDestroy is not after a FrameSetup", &I);
35400b57cec5SDimitry Andric int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
35410b57cec5SDimitry Andric BBState.ExitValue;
35420b57cec5SDimitry Andric if (BBState.ExitIsSetup && AbsSPAdj != Size) {
35430b57cec5SDimitry Andric report("FrameDestroy <n> is after FrameSetup <m>", &I);
35440b57cec5SDimitry Andric errs() << "FrameDestroy <" << Size << "> is after FrameSetup <"
35450b57cec5SDimitry Andric << AbsSPAdj << ">.\n";
35460b57cec5SDimitry Andric }
35470b57cec5SDimitry Andric BBState.ExitValue += Size;
35480b57cec5SDimitry Andric BBState.ExitIsSetup = false;
35490b57cec5SDimitry Andric }
35500b57cec5SDimitry Andric }
35510b57cec5SDimitry Andric SPState[MBB->getNumber()] = BBState;
35520b57cec5SDimitry Andric
35530b57cec5SDimitry Andric // Make sure the exit state of any predecessor is consistent with the entry
35540b57cec5SDimitry Andric // state.
35555ffd83dbSDimitry Andric for (const MachineBasicBlock *Pred : MBB->predecessors()) {
35565ffd83dbSDimitry Andric if (Reachable.count(Pred) &&
35575ffd83dbSDimitry Andric (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
35585ffd83dbSDimitry Andric SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
35590b57cec5SDimitry Andric report("The exit stack state of a predecessor is inconsistent.", MBB);
35605ffd83dbSDimitry Andric errs() << "Predecessor " << printMBBReference(*Pred)
35615ffd83dbSDimitry Andric << " has exit state (" << SPState[Pred->getNumber()].ExitValue
35625ffd83dbSDimitry Andric << ", " << SPState[Pred->getNumber()].ExitIsSetup << "), while "
35630b57cec5SDimitry Andric << printMBBReference(*MBB) << " has entry state ("
35640b57cec5SDimitry Andric << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
35650b57cec5SDimitry Andric }
35660b57cec5SDimitry Andric }
35670b57cec5SDimitry Andric
35680b57cec5SDimitry Andric // Make sure the entry state of any successor is consistent with the exit
35690b57cec5SDimitry Andric // state.
35705ffd83dbSDimitry Andric for (const MachineBasicBlock *Succ : MBB->successors()) {
35715ffd83dbSDimitry Andric if (Reachable.count(Succ) &&
35725ffd83dbSDimitry Andric (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
35735ffd83dbSDimitry Andric SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
35740b57cec5SDimitry Andric report("The entry stack state of a successor is inconsistent.", MBB);
35755ffd83dbSDimitry Andric errs() << "Successor " << printMBBReference(*Succ)
35765ffd83dbSDimitry Andric << " has entry state (" << SPState[Succ->getNumber()].EntryValue
35775ffd83dbSDimitry Andric << ", " << SPState[Succ->getNumber()].EntryIsSetup << "), while "
35780b57cec5SDimitry Andric << printMBBReference(*MBB) << " has exit state ("
35790b57cec5SDimitry Andric << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
35800b57cec5SDimitry Andric }
35810b57cec5SDimitry Andric }
35820b57cec5SDimitry Andric
35830b57cec5SDimitry Andric // Make sure a basic block with return ends with zero stack adjustment.
35840b57cec5SDimitry Andric if (!MBB->empty() && MBB->back().isReturn()) {
35850b57cec5SDimitry Andric if (BBState.ExitIsSetup)
35860b57cec5SDimitry Andric report("A return block ends with a FrameSetup.", MBB);
35870b57cec5SDimitry Andric if (BBState.ExitValue)
35880b57cec5SDimitry Andric report("A return block ends with a nonzero stack adjustment.", MBB);
35890b57cec5SDimitry Andric }
35900b57cec5SDimitry Andric }
35910b57cec5SDimitry Andric }
3592