1 //===-- SIModeRegister.cpp - Mode Register --------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This pass inserts changes to the Mode register settings as required. 10 /// Note that currently it only deals with the Double Precision Floating Point 11 /// rounding mode setting, but is intended to be generic enough to be easily 12 /// expanded. 13 /// 14 //===----------------------------------------------------------------------===// 15 // 16 #include "AMDGPU.h" 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUSubtarget.h" 19 #include "SIInstrInfo.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/ADT/Statistic.h" 22 #include "llvm/CodeGen/MachineFunctionPass.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineRegisterInfo.h" 25 #include "llvm/IR/Constants.h" 26 #include "llvm/IR/Function.h" 27 #include "llvm/IR/LLVMContext.h" 28 #include "llvm/Support/Debug.h" 29 #include "llvm/Support/raw_ostream.h" 30 #include "llvm/Target/TargetMachine.h" 31 #include <queue> 32 33 #define DEBUG_TYPE "si-mode-register" 34 35 STATISTIC(NumSetregInserted, "Number of setreg of mode register inserted."); 36 37 using namespace llvm; 38 39 struct Status { 40 // Mask is a bitmask where a '1' indicates the corresponding Mode bit has a 41 // known value 42 unsigned Mask; 43 unsigned Mode; 44 45 Status() : Mask(0), Mode(0){}; 46 47 Status(unsigned NewMask, unsigned NewMode) : Mask(NewMask), Mode(NewMode) { 48 Mode &= Mask; 49 }; 50 51 // merge two status values such that only values that don't conflict are 52 // preserved 53 Status merge(const Status &S) const { 54 return Status((Mask | S.Mask), ((Mode & ~S.Mask) | (S.Mode & S.Mask))); 55 } 56 57 // merge an unknown value by using the unknown value's mask to remove bits 58 // from the result 59 Status mergeUnknown(unsigned newMask) { 60 return Status(Mask & ~newMask, Mode & ~newMask); 61 } 62 63 // intersect two Status values to produce a mode and mask that is a subset 64 // of both values 65 Status intersect(const Status &S) const { 66 unsigned NewMask = (Mask & S.Mask) & (Mode ^ ~S.Mode); 67 unsigned NewMode = (Mode & NewMask); 68 return Status(NewMask, NewMode); 69 } 70 71 // produce the delta required to change the Mode to the required Mode 72 Status delta(const Status &S) const { 73 return Status((S.Mask & (Mode ^ S.Mode)) | (~Mask & S.Mask), S.Mode); 74 } 75 76 bool operator==(const Status &S) const { 77 return (Mask == S.Mask) && (Mode == S.Mode); 78 } 79 80 bool operator!=(const Status &S) const { return !(*this == S); } 81 82 bool isCompatible(Status &S) { 83 return ((Mask & S.Mask) == S.Mask) && ((Mode & S.Mask) == S.Mode); 84 } 85 86 bool isCombinable(Status &S) { return !(Mask & S.Mask) || isCompatible(S); } 87 }; 88 89 class BlockData { 90 public: 91 // The Status that represents the mode register settings required by the 92 // FirstInsertionPoint (if any) in this block. Calculated in Phase 1. 93 Status Require; 94 95 // The Status that represents the net changes to the Mode register made by 96 // this block, Calculated in Phase 1. 97 Status Change; 98 99 // The Status that represents the mode register settings on exit from this 100 // block. Calculated in Phase 2. 101 Status Exit; 102 103 // The Status that represents the intersection of exit Mode register settings 104 // from all predecessor blocks. Calculated in Phase 2, and used by Phase 3. 105 Status Pred; 106 107 // In Phase 1 we record the first instruction that has a mode requirement, 108 // which is used in Phase 3 if we need to insert a mode change. 109 MachineInstr *FirstInsertionPoint; 110 111 // A flag to indicate whether an Exit value has been set (we can't tell by 112 // examining the Exit value itself as all values may be valid results). 113 bool ExitSet; 114 115 BlockData() : FirstInsertionPoint(nullptr), ExitSet(false){}; 116 }; 117 118 namespace { 119 120 class SIModeRegister : public MachineFunctionPass { 121 public: 122 static char ID; 123 124 std::vector<std::unique_ptr<BlockData>> BlockInfo; 125 std::queue<MachineBasicBlock *> Phase2List; 126 127 // The default mode register setting currently only caters for the floating 128 // point double precision rounding mode. 129 // We currently assume the default rounding mode is Round to Nearest 130 // NOTE: this should come from a per function rounding mode setting once such 131 // a setting exists. 132 unsigned DefaultMode = FP_ROUND_ROUND_TO_NEAREST; 133 Status DefaultStatus = 134 Status(FP_ROUND_MODE_DP(0x3), FP_ROUND_MODE_DP(DefaultMode)); 135 136 bool Changed = false; 137 138 public: 139 SIModeRegister() : MachineFunctionPass(ID) {} 140 141 bool runOnMachineFunction(MachineFunction &MF) override; 142 143 void getAnalysisUsage(AnalysisUsage &AU) const override { 144 AU.setPreservesCFG(); 145 MachineFunctionPass::getAnalysisUsage(AU); 146 } 147 148 void processBlockPhase1(MachineBasicBlock &MBB, const SIInstrInfo *TII); 149 150 void processBlockPhase2(MachineBasicBlock &MBB, const SIInstrInfo *TII); 151 152 void processBlockPhase3(MachineBasicBlock &MBB, const SIInstrInfo *TII); 153 154 Status getInstructionMode(MachineInstr &MI, const SIInstrInfo *TII); 155 156 void insertSetreg(MachineBasicBlock &MBB, MachineInstr *I, 157 const SIInstrInfo *TII, Status InstrMode); 158 }; 159 } // End anonymous namespace. 160 161 INITIALIZE_PASS(SIModeRegister, DEBUG_TYPE, 162 "Insert required mode register values", false, false) 163 164 char SIModeRegister::ID = 0; 165 166 char &llvm::SIModeRegisterID = SIModeRegister::ID; 167 168 FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); } 169 170 // Determine the Mode register setting required for this instruction. 171 // Instructions which don't use the Mode register return a null Status. 172 // Note this currently only deals with instructions that use the floating point 173 // double precision setting. 174 Status SIModeRegister::getInstructionMode(MachineInstr &MI, 175 const SIInstrInfo *TII) { 176 if (TII->usesFPDPRounding(MI)) { 177 switch (MI.getOpcode()) { 178 case AMDGPU::V_INTERP_P1LL_F16: 179 case AMDGPU::V_INTERP_P1LV_F16: 180 case AMDGPU::V_INTERP_P2_F16: 181 // f16 interpolation instructions need double precision round to zero 182 return Status(FP_ROUND_MODE_DP(3), 183 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO)); 184 default: 185 return DefaultStatus; 186 } 187 } 188 return Status(); 189 } 190 191 // Insert a setreg instruction to update the Mode register. 192 // It is possible (though unlikely) for an instruction to require a change to 193 // the value of disjoint parts of the Mode register when we don't know the 194 // value of the intervening bits. In that case we need to use more than one 195 // setreg instruction. 196 void SIModeRegister::insertSetreg(MachineBasicBlock &MBB, MachineInstr *MI, 197 const SIInstrInfo *TII, Status InstrMode) { 198 while (InstrMode.Mask) { 199 unsigned Offset = countTrailingZeros<unsigned>(InstrMode.Mask); 200 unsigned Width = countTrailingOnes<unsigned>(InstrMode.Mask >> Offset); 201 unsigned Value = (InstrMode.Mode >> Offset) & ((1 << Width) - 1); 202 BuildMI(MBB, MI, 0, TII->get(AMDGPU::S_SETREG_IMM32_B32)) 203 .addImm(Value) 204 .addImm(((Width - 1) << AMDGPU::Hwreg::WIDTH_M1_SHIFT_) | 205 (Offset << AMDGPU::Hwreg::OFFSET_SHIFT_) | 206 (AMDGPU::Hwreg::ID_MODE << AMDGPU::Hwreg::ID_SHIFT_)); 207 ++NumSetregInserted; 208 Changed = true; 209 InstrMode.Mask &= ~(((1 << Width) - 1) << Offset); 210 } 211 } 212 213 // In Phase 1 we iterate through the instructions of the block and for each 214 // instruction we get its mode usage. If the instruction uses the Mode register 215 // we: 216 // - update the Change status, which tracks the changes to the Mode register 217 // made by this block 218 // - if this instruction's requirements are compatible with the current setting 219 // of the Mode register we merge the modes 220 // - if it isn't compatible and an InsertionPoint isn't set, then we set the 221 // InsertionPoint to the current instruction, and we remember the current 222 // mode 223 // - if it isn't compatible and InsertionPoint is set we insert a seteg before 224 // that instruction (unless this instruction forms part of the block's 225 // entry requirements in which case the insertion is deferred until Phase 3 226 // when predecessor exit values are known), and move the insertion point to 227 // this instruction 228 // - if this is a setreg instruction we treat it as an incompatible instruction. 229 // This is sub-optimal but avoids some nasty corner cases, and is expected to 230 // occur very rarely. 231 // - on exit we have set the Require, Change, and initial Exit modes. 232 void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB, 233 const SIInstrInfo *TII) { 234 auto NewInfo = std::make_unique<BlockData>(); 235 MachineInstr *InsertionPoint = nullptr; 236 // RequirePending is used to indicate whether we are collecting the initial 237 // requirements for the block, and need to defer the first InsertionPoint to 238 // Phase 3. It is set to false once we have set FirstInsertionPoint, or when 239 // we discover an explict setreg that means this block doesn't have any 240 // initial requirements. 241 bool RequirePending = true; 242 Status IPChange; 243 for (MachineInstr &MI : MBB) { 244 Status InstrMode = getInstructionMode(MI, TII); 245 if ((MI.getOpcode() == AMDGPU::S_SETREG_B32) || 246 (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32)) { 247 // We preserve any explicit mode register setreg instruction we encounter, 248 // as we assume it has been inserted by a higher authority (this is 249 // likely to be a very rare occurrence). 250 unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm(); 251 if (((Dst & AMDGPU::Hwreg::ID_MASK_) >> AMDGPU::Hwreg::ID_SHIFT_) != 252 AMDGPU::Hwreg::ID_MODE) 253 continue; 254 255 unsigned Width = ((Dst & AMDGPU::Hwreg::WIDTH_M1_MASK_) >> 256 AMDGPU::Hwreg::WIDTH_M1_SHIFT_) + 257 1; 258 unsigned Offset = 259 (Dst & AMDGPU::Hwreg::OFFSET_MASK_) >> AMDGPU::Hwreg::OFFSET_SHIFT_; 260 unsigned Mask = ((1 << Width) - 1) << Offset; 261 262 // If an InsertionPoint is set we will insert a setreg there. 263 if (InsertionPoint) { 264 insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change)); 265 InsertionPoint = nullptr; 266 } 267 // If this is an immediate then we know the value being set, but if it is 268 // not an immediate then we treat the modified bits of the mode register 269 // as unknown. 270 if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32) { 271 unsigned Val = TII->getNamedOperand(MI, AMDGPU::OpName::imm)->getImm(); 272 unsigned Mode = (Val << Offset) & Mask; 273 Status Setreg = Status(Mask, Mode); 274 // If we haven't already set the initial requirements for the block we 275 // don't need to as the requirements start from this explicit setreg. 276 RequirePending = false; 277 NewInfo->Change = NewInfo->Change.merge(Setreg); 278 } else { 279 NewInfo->Change = NewInfo->Change.mergeUnknown(Mask); 280 } 281 } else if (!NewInfo->Change.isCompatible(InstrMode)) { 282 // This instruction uses the Mode register and its requirements aren't 283 // compatible with the current mode. 284 if (InsertionPoint) { 285 // If the required mode change cannot be included in the current 286 // InsertionPoint changes, we need a setreg and start a new 287 // InsertionPoint. 288 if (!IPChange.delta(NewInfo->Change).isCombinable(InstrMode)) { 289 if (RequirePending) { 290 // This is the first insertionPoint in the block so we will defer 291 // the insertion of the setreg to Phase 3 where we know whether or 292 // not it is actually needed. 293 NewInfo->FirstInsertionPoint = InsertionPoint; 294 NewInfo->Require = NewInfo->Change; 295 RequirePending = false; 296 } else { 297 insertSetreg(MBB, InsertionPoint, TII, 298 IPChange.delta(NewInfo->Change)); 299 IPChange = NewInfo->Change; 300 } 301 // Set the new InsertionPoint 302 InsertionPoint = &MI; 303 } 304 NewInfo->Change = NewInfo->Change.merge(InstrMode); 305 } else { 306 // No InsertionPoint is currently set - this is either the first in 307 // the block or we have previously seen an explicit setreg. 308 InsertionPoint = &MI; 309 IPChange = NewInfo->Change; 310 NewInfo->Change = NewInfo->Change.merge(InstrMode); 311 } 312 } 313 } 314 if (RequirePending) { 315 // If we haven't yet set the initial requirements for the block we set them 316 // now. 317 NewInfo->FirstInsertionPoint = InsertionPoint; 318 NewInfo->Require = NewInfo->Change; 319 } else if (InsertionPoint) { 320 // We need to insert a setreg at the InsertionPoint 321 insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change)); 322 } 323 NewInfo->Exit = NewInfo->Change; 324 BlockInfo[MBB.getNumber()] = std::move(NewInfo); 325 } 326 327 // In Phase 2 we revisit each block and calculate the common Mode register 328 // value provided by all predecessor blocks. If the Exit value for the block 329 // is changed, then we add the successor blocks to the worklist so that the 330 // exit value is propagated. 331 void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB, 332 const SIInstrInfo *TII) { 333 bool RevisitRequired = false; 334 bool ExitSet = false; 335 unsigned ThisBlock = MBB.getNumber(); 336 if (MBB.pred_empty()) { 337 // There are no predecessors, so use the default starting status. 338 BlockInfo[ThisBlock]->Pred = DefaultStatus; 339 ExitSet = true; 340 } else { 341 // Build a status that is common to all the predecessors by intersecting 342 // all the predecessor exit status values. 343 // Mask bits (which represent the Mode bits with a known value) can only be 344 // added by explicit SETREG instructions or the initial default value - 345 // the intersection process may remove Mask bits. 346 // If we find a predecessor that has not yet had an exit value determined 347 // (this can happen for example if a block is its own predecessor) we defer 348 // use of that value as the Mask will be all zero, and we will revisit this 349 // block again later (unless the only predecessor without an exit value is 350 // this block). 351 MachineBasicBlock::pred_iterator P = MBB.pred_begin(), E = MBB.pred_end(); 352 MachineBasicBlock &PB = *(*P); 353 unsigned PredBlock = PB.getNumber(); 354 if ((ThisBlock == PredBlock) && (std::next(P) == E)) { 355 BlockInfo[ThisBlock]->Pred = DefaultStatus; 356 ExitSet = true; 357 } else if (BlockInfo[PredBlock]->ExitSet) { 358 BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit; 359 ExitSet = true; 360 } else if (PredBlock != ThisBlock) 361 RevisitRequired = true; 362 363 for (P = std::next(P); P != E; P = std::next(P)) { 364 MachineBasicBlock *Pred = *P; 365 unsigned PredBlock = Pred->getNumber(); 366 if (BlockInfo[PredBlock]->ExitSet) { 367 if (BlockInfo[ThisBlock]->ExitSet) { 368 BlockInfo[ThisBlock]->Pred = 369 BlockInfo[ThisBlock]->Pred.intersect(BlockInfo[PredBlock]->Exit); 370 } else { 371 BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit; 372 } 373 ExitSet = true; 374 } else if (PredBlock != ThisBlock) 375 RevisitRequired = true; 376 } 377 } 378 Status TmpStatus = 379 BlockInfo[ThisBlock]->Pred.merge(BlockInfo[ThisBlock]->Change); 380 if (BlockInfo[ThisBlock]->Exit != TmpStatus) { 381 BlockInfo[ThisBlock]->Exit = TmpStatus; 382 // Add the successors to the work list so we can propagate the changed exit 383 // status. 384 for (MachineBasicBlock::succ_iterator S = MBB.succ_begin(), 385 E = MBB.succ_end(); 386 S != E; S = std::next(S)) { 387 MachineBasicBlock &B = *(*S); 388 Phase2List.push(&B); 389 } 390 } 391 BlockInfo[ThisBlock]->ExitSet = ExitSet; 392 if (RevisitRequired) 393 Phase2List.push(&MBB); 394 } 395 396 // In Phase 3 we revisit each block and if it has an insertion point defined we 397 // check whether the predecessor mode meets the block's entry requirements. If 398 // not we insert an appropriate setreg instruction to modify the Mode register. 399 void SIModeRegister::processBlockPhase3(MachineBasicBlock &MBB, 400 const SIInstrInfo *TII) { 401 unsigned ThisBlock = MBB.getNumber(); 402 if (!BlockInfo[ThisBlock]->Pred.isCompatible(BlockInfo[ThisBlock]->Require)) { 403 Status Delta = 404 BlockInfo[ThisBlock]->Pred.delta(BlockInfo[ThisBlock]->Require); 405 if (BlockInfo[ThisBlock]->FirstInsertionPoint) 406 insertSetreg(MBB, BlockInfo[ThisBlock]->FirstInsertionPoint, TII, Delta); 407 else 408 insertSetreg(MBB, &MBB.instr_front(), TII, Delta); 409 } 410 } 411 412 bool SIModeRegister::runOnMachineFunction(MachineFunction &MF) { 413 BlockInfo.resize(MF.getNumBlockIDs()); 414 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 415 const SIInstrInfo *TII = ST.getInstrInfo(); 416 417 // Processing is performed in a number of phases 418 419 // Phase 1 - determine the initial mode required by each block, and add setreg 420 // instructions for intra block requirements. 421 for (MachineBasicBlock &BB : MF) 422 processBlockPhase1(BB, TII); 423 424 // Phase 2 - determine the exit mode from each block. We add all blocks to the 425 // list here, but will also add any that need to be revisited during Phase 2 426 // processing. 427 for (MachineBasicBlock &BB : MF) 428 Phase2List.push(&BB); 429 while (!Phase2List.empty()) { 430 processBlockPhase2(*Phase2List.front(), TII); 431 Phase2List.pop(); 432 } 433 434 // Phase 3 - add an initial setreg to each block where the required entry mode 435 // is not satisfied by the exit mode of all its predecessors. 436 for (MachineBasicBlock &BB : MF) 437 processBlockPhase3(BB, TII); 438 439 BlockInfo.clear(); 440 441 return Changed; 442 } 443