1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineCombinerPattern.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFrameInfo.h"
28 #include "llvm/CodeGen/MachineFunctionPass.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineMemOperand.h"
31 #include "llvm/CodeGen/MachineRegisterInfo.h"
32 #include "llvm/CodeGen/PseudoSourceValue.h"
33 #include "llvm/CodeGen/RegisterClassInfo.h"
34 #include "llvm/CodeGen/RegisterPressure.h"
35 #include "llvm/CodeGen/ScheduleDAG.h"
36 #include "llvm/CodeGen/SlotIndexes.h"
37 #include "llvm/CodeGen/StackMaps.h"
38 #include "llvm/MC/MCAsmInfo.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/TargetRegistry.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/raw_ostream.h"
45
46 using namespace llvm;
47
48 #define DEBUG_TYPE "ppc-instr-info"
49
50 #define GET_INSTRMAP_INFO
51 #define GET_INSTRINFO_CTOR_DTOR
52 #include "PPCGenInstrInfo.inc"
53
54 STATISTIC(NumStoreSPILLVSRRCAsVec,
55 "Number of spillvsrrc spilled to stack as vec");
56 STATISTIC(NumStoreSPILLVSRRCAsGpr,
57 "Number of spillvsrrc spilled to stack as gpr");
58 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
59 STATISTIC(CmpIselsConverted,
60 "Number of ISELs that depend on comparison of constants converted");
61 STATISTIC(MissedConvertibleImmediateInstrs,
62 "Number of compare-immediate instructions fed by constants");
63 STATISTIC(NumRcRotatesConvertedToRcAnd,
64 "Number of record-form rotates converted to record-form andi");
65
66 static cl::
67 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
68 cl::desc("Disable analysis for CTR loops"));
69
70 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
71 cl::desc("Disable compare instruction optimization"), cl::Hidden);
72
73 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
74 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
75 cl::Hidden);
76
77 static cl::opt<bool>
78 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
79 cl::desc("Use the old (incorrect) instruction latency calculation"));
80
81 static cl::opt<float>
82 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
83 cl::desc("register pressure factor for the transformations."));
84
85 static cl::opt<bool> EnableFMARegPressureReduction(
86 "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
87 cl::desc("enable register pressure reduce in machine combiner pass."));
88
89 // Pin the vtable to this file.
anchor()90 void PPCInstrInfo::anchor() {}
91
PPCInstrInfo(PPCSubtarget & STI)92 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
93 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
94 /* CatchRetOpcode */ -1,
95 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
96 Subtarget(STI), RI(STI.getTargetMachine()) {}
97
98 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
99 /// this target when scheduling the DAG.
100 ScheduleHazardRecognizer *
CreateTargetHazardRecognizer(const TargetSubtargetInfo * STI,const ScheduleDAG * DAG) const101 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
102 const ScheduleDAG *DAG) const {
103 unsigned Directive =
104 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
105 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
106 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
107 const InstrItineraryData *II =
108 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
109 return new ScoreboardHazardRecognizer(II, DAG);
110 }
111
112 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
113 }
114
115 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
116 /// to use for this target when scheduling the DAG.
117 ScheduleHazardRecognizer *
CreateTargetPostRAHazardRecognizer(const InstrItineraryData * II,const ScheduleDAG * DAG) const118 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
119 const ScheduleDAG *DAG) const {
120 unsigned Directive =
121 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
122
123 // FIXME: Leaving this as-is until we have POWER9 scheduling info
124 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
125 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
126
127 // Most subtargets use a PPC970 recognizer.
128 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
129 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
130 assert(DAG->TII && "No InstrInfo?");
131
132 return new PPCHazardRecognizer970(*DAG);
133 }
134
135 return new ScoreboardHazardRecognizer(II, DAG);
136 }
137
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr & MI,unsigned * PredCost) const138 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
139 const MachineInstr &MI,
140 unsigned *PredCost) const {
141 if (!ItinData || UseOldLatencyCalc)
142 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
143
144 // The default implementation of getInstrLatency calls getStageLatency, but
145 // getStageLatency does not do the right thing for us. While we have
146 // itinerary, most cores are fully pipelined, and so the itineraries only
147 // express the first part of the pipeline, not every stage. Instead, we need
148 // to use the listed output operand cycle number (using operand 0 here, which
149 // is an output).
150
151 unsigned Latency = 1;
152 unsigned DefClass = MI.getDesc().getSchedClass();
153 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
154 const MachineOperand &MO = MI.getOperand(i);
155 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
156 continue;
157
158 std::optional<unsigned> Cycle = ItinData->getOperandCycle(DefClass, i);
159 if (!Cycle)
160 continue;
161
162 Latency = std::max(Latency, *Cycle);
163 }
164
165 return Latency;
166 }
167
getOperandLatency(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const168 std::optional<unsigned> PPCInstrInfo::getOperandLatency(
169 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
170 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
171 std::optional<unsigned> Latency = PPCGenInstrInfo::getOperandLatency(
172 ItinData, DefMI, DefIdx, UseMI, UseIdx);
173
174 if (!DefMI.getParent())
175 return Latency;
176
177 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
178 Register Reg = DefMO.getReg();
179
180 bool IsRegCR;
181 if (Reg.isVirtual()) {
182 const MachineRegisterInfo *MRI =
183 &DefMI.getParent()->getParent()->getRegInfo();
184 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
185 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
186 } else {
187 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
188 PPC::CRBITRCRegClass.contains(Reg);
189 }
190
191 if (UseMI.isBranch() && IsRegCR) {
192 if (!Latency)
193 Latency = getInstrLatency(ItinData, DefMI);
194
195 // On some cores, there is an additional delay between writing to a condition
196 // register, and using it from a branch.
197 unsigned Directive = Subtarget.getCPUDirective();
198 switch (Directive) {
199 default: break;
200 case PPC::DIR_7400:
201 case PPC::DIR_750:
202 case PPC::DIR_970:
203 case PPC::DIR_E5500:
204 case PPC::DIR_PWR4:
205 case PPC::DIR_PWR5:
206 case PPC::DIR_PWR5X:
207 case PPC::DIR_PWR6:
208 case PPC::DIR_PWR6X:
209 case PPC::DIR_PWR7:
210 case PPC::DIR_PWR8:
211 // FIXME: Is this needed for POWER9?
212 Latency = *Latency + 2;
213 break;
214 }
215 }
216
217 return Latency;
218 }
219
setSpecialOperandAttr(MachineInstr & MI,uint32_t Flags) const220 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
221 uint32_t Flags) const {
222 MI.setFlags(Flags);
223 MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
224 MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
225 MI.clearFlag(MachineInstr::MIFlag::IsExact);
226 }
227
228 // This function does not list all associative and commutative operations, but
229 // only those worth feeding through the machine combiner in an attempt to
230 // reduce the critical path. Mostly, this means floating-point operations,
231 // because they have high latencies(>=5) (compared to other operations, such as
232 // and/or, which are also associative and commutative, but have low latencies).
isAssociativeAndCommutative(const MachineInstr & Inst,bool Invert) const233 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
234 bool Invert) const {
235 if (Invert)
236 return false;
237 switch (Inst.getOpcode()) {
238 // Floating point:
239 // FP Add:
240 case PPC::FADD:
241 case PPC::FADDS:
242 // FP Multiply:
243 case PPC::FMUL:
244 case PPC::FMULS:
245 // Altivec Add:
246 case PPC::VADDFP:
247 // VSX Add:
248 case PPC::XSADDDP:
249 case PPC::XVADDDP:
250 case PPC::XVADDSP:
251 case PPC::XSADDSP:
252 // VSX Multiply:
253 case PPC::XSMULDP:
254 case PPC::XVMULDP:
255 case PPC::XVMULSP:
256 case PPC::XSMULSP:
257 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
258 Inst.getFlag(MachineInstr::MIFlag::FmNsz);
259 // Fixed point:
260 // Multiply:
261 case PPC::MULHD:
262 case PPC::MULLD:
263 case PPC::MULHW:
264 case PPC::MULLW:
265 return true;
266 default:
267 return false;
268 }
269 }
270
271 #define InfoArrayIdxFMAInst 0
272 #define InfoArrayIdxFAddInst 1
273 #define InfoArrayIdxFMULInst 2
274 #define InfoArrayIdxAddOpIdx 3
275 #define InfoArrayIdxMULOpIdx 4
276 #define InfoArrayIdxFSubInst 5
277 // Array keeps info for FMA instructions:
278 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
279 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
280 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
281 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
282 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
283 // second MUL operand index is plus 1;
284 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
285 static const uint16_t FMAOpIdxInfo[][6] = {
286 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
287 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
288 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
289 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
290 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
291 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
292 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
293
294 // Check if an opcode is a FMA instruction. If it is, return the index in array
295 // FMAOpIdxInfo. Otherwise, return -1.
getFMAOpIdxInfo(unsigned Opcode) const296 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
297 for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
298 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
299 return I;
300 return -1;
301 }
302
303 // On PowerPC target, we have two kinds of patterns related to FMA:
304 // 1: Improve ILP.
305 // Try to reassociate FMA chains like below:
306 //
307 // Pattern 1:
308 // A = FADD X, Y (Leaf)
309 // B = FMA A, M21, M22 (Prev)
310 // C = FMA B, M31, M32 (Root)
311 // -->
312 // A = FMA X, M21, M22
313 // B = FMA Y, M31, M32
314 // C = FADD A, B
315 //
316 // Pattern 2:
317 // A = FMA X, M11, M12 (Leaf)
318 // B = FMA A, M21, M22 (Prev)
319 // C = FMA B, M31, M32 (Root)
320 // -->
321 // A = FMUL M11, M12
322 // B = FMA X, M21, M22
323 // D = FMA A, M31, M32
324 // C = FADD B, D
325 //
326 // breaking the dependency between A and B, allowing FMA to be executed in
327 // parallel (or back-to-back in a pipeline) instead of depending on each other.
328 //
329 // 2: Reduce register pressure.
330 // Try to reassociate FMA with FSUB and a constant like below:
331 // C is a floating point const.
332 //
333 // Pattern 1:
334 // A = FSUB X, Y (Leaf)
335 // D = FMA B, C, A (Root)
336 // -->
337 // A = FMA B, Y, -C
338 // D = FMA A, X, C
339 //
340 // Pattern 2:
341 // A = FSUB X, Y (Leaf)
342 // D = FMA B, A, C (Root)
343 // -->
344 // A = FMA B, Y, -C
345 // D = FMA A, X, C
346 //
347 // Before the transformation, A must be assigned with different hardware
348 // register with D. After the transformation, A and D must be assigned with
349 // same hardware register due to TIE attribute of FMA instructions.
350 //
getFMAPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns,bool DoRegPressureReduce) const351 bool PPCInstrInfo::getFMAPatterns(
352 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
353 bool DoRegPressureReduce) const {
354 MachineBasicBlock *MBB = Root.getParent();
355 const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
356 const TargetRegisterInfo *TRI = &getRegisterInfo();
357
358 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
359 for (const auto &MO : Instr.explicit_operands())
360 if (!(MO.isReg() && MO.getReg().isVirtual()))
361 return false;
362 return true;
363 };
364
365 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
366 unsigned OpType) {
367 if (Instr.getOpcode() !=
368 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
369 return false;
370
371 // Instruction can be reassociated.
372 // fast math flags may prohibit reassociation.
373 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
374 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
375 return false;
376
377 // Instruction operands are virtual registers for reassociation.
378 if (!IsAllOpsVirtualReg(Instr))
379 return false;
380
381 // For register pressure reassociation, the FSub must have only one use as
382 // we want to delete the sub to save its def.
383 if (OpType == InfoArrayIdxFSubInst &&
384 !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
385 return false;
386
387 return true;
388 };
389
390 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
391 int16_t &MulOpIdx, bool IsLeaf) {
392 int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
393 if (Idx < 0)
394 return false;
395
396 // Instruction can be reassociated.
397 // fast math flags may prohibit reassociation.
398 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
399 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
400 return false;
401
402 // Instruction operands are virtual registers for reassociation.
403 if (!IsAllOpsVirtualReg(Instr))
404 return false;
405
406 MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
407 if (IsLeaf)
408 return true;
409
410 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
411
412 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
413 MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
414 // If 'add' operand's def is not in current block, don't do ILP related opt.
415 if (!MIAdd || MIAdd->getParent() != MBB)
416 return false;
417
418 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
419 // as this fma will be changed later.
420 return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
421 };
422
423 int16_t AddOpIdx = -1;
424 int16_t MulOpIdx = -1;
425
426 bool IsUsedOnceL = false;
427 bool IsUsedOnceR = false;
428 MachineInstr *MULInstrL = nullptr;
429 MachineInstr *MULInstrR = nullptr;
430
431 auto IsRPReductionCandidate = [&]() {
432 // Currently, we only support float and double.
433 // FIXME: add support for other types.
434 unsigned Opcode = Root.getOpcode();
435 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
436 return false;
437
438 // Root must be a valid FMA like instruction.
439 // Treat it as leaf as we don't care its add operand.
440 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
441 assert((MulOpIdx >= 0) && "mul operand index not right!");
442 Register MULRegL = TRI->lookThruSingleUseCopyChain(
443 Root.getOperand(MulOpIdx).getReg(), MRI);
444 Register MULRegR = TRI->lookThruSingleUseCopyChain(
445 Root.getOperand(MulOpIdx + 1).getReg(), MRI);
446 if (!MULRegL && !MULRegR)
447 return false;
448
449 if (MULRegL && !MULRegR) {
450 MULRegR =
451 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
452 IsUsedOnceL = true;
453 } else if (!MULRegL && MULRegR) {
454 MULRegL =
455 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
456 IsUsedOnceR = true;
457 } else {
458 IsUsedOnceL = true;
459 IsUsedOnceR = true;
460 }
461
462 if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
463 return false;
464
465 MULInstrL = MRI->getVRegDef(MULRegL);
466 MULInstrR = MRI->getVRegDef(MULRegR);
467 return true;
468 }
469 return false;
470 };
471
472 // Register pressure fma reassociation patterns.
473 if (DoRegPressureReduce && IsRPReductionCandidate()) {
474 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
475 // Register pressure pattern 1
476 if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
477 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
478 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
479 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA);
480 return true;
481 }
482
483 // Register pressure pattern 2
484 if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
485 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
486 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
487 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC);
488 return true;
489 }
490 }
491
492 // ILP fma reassociation patterns.
493 // Root must be a valid FMA like instruction.
494 AddOpIdx = -1;
495 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
496 return false;
497
498 assert((AddOpIdx >= 0) && "add operand index not right!");
499
500 Register RegB = Root.getOperand(AddOpIdx).getReg();
501 MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
502
503 // Prev must be a valid FMA like instruction.
504 AddOpIdx = -1;
505 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
506 return false;
507
508 assert((AddOpIdx >= 0) && "add operand index not right!");
509
510 Register RegA = Prev->getOperand(AddOpIdx).getReg();
511 MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
512 AddOpIdx = -1;
513 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
514 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
515 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
516 return true;
517 }
518 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
519 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
520 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
521 return true;
522 }
523 return false;
524 }
525
finalizeInsInstrs(MachineInstr & Root,MachineCombinerPattern & P,SmallVectorImpl<MachineInstr * > & InsInstrs) const526 void PPCInstrInfo::finalizeInsInstrs(
527 MachineInstr &Root, MachineCombinerPattern &P,
528 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
529 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
530
531 MachineFunction *MF = Root.getMF();
532 MachineRegisterInfo *MRI = &MF->getRegInfo();
533 const TargetRegisterInfo *TRI = &getRegisterInfo();
534 MachineConstantPool *MCP = MF->getConstantPool();
535
536 int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
537 if (Idx < 0)
538 return;
539
540 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
541
542 // For now we only need to fix up placeholder for register pressure reduce
543 // patterns.
544 Register ConstReg = 0;
545 switch (P) {
546 case MachineCombinerPattern::REASSOC_XY_BCA:
547 ConstReg =
548 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
549 break;
550 case MachineCombinerPattern::REASSOC_XY_BAC:
551 ConstReg =
552 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
553 break;
554 default:
555 // Not register pressure reduce patterns.
556 return;
557 }
558
559 MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
560 // Get const value from const pool.
561 const Constant *C = getConstantFromConstantPool(ConstDefInstr);
562 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
563
564 // Get negative fp const.
565 APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
566 F1.changeSign();
567 Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
568 Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
569
570 // Put negative fp const into constant pool.
571 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
572
573 MachineOperand *Placeholder = nullptr;
574 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
575 for (auto *Inst : InsInstrs) {
576 for (MachineOperand &Operand : Inst->explicit_operands()) {
577 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
578 if (Operand.getReg() == PPC::ZERO8) {
579 Placeholder = &Operand;
580 break;
581 }
582 }
583 }
584
585 assert(Placeholder && "Placeholder does not exist!");
586
587 // Generate instructions to load the const fp from constant pool.
588 // We only support PPC64 and medium code model.
589 Register LoadNewConst =
590 generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
591
592 // Fill the placeholder with the new load from constant pool.
593 Placeholder->setReg(LoadNewConst);
594 }
595
shouldReduceRegisterPressure(const MachineBasicBlock * MBB,const RegisterClassInfo * RegClassInfo) const596 bool PPCInstrInfo::shouldReduceRegisterPressure(
597 const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
598
599 if (!EnableFMARegPressureReduction)
600 return false;
601
602 // Currently, we only enable register pressure reducing in machine combiner
603 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
604 // support.
605 //
606 // So we need following instructions to access a TOC entry:
607 //
608 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
609 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
610 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
611 //
612 // FIXME: add more supported targets, like Small and Large code model, PPC32,
613 // AIX.
614 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
615 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
616 return false;
617
618 const TargetRegisterInfo *TRI = &getRegisterInfo();
619 const MachineFunction *MF = MBB->getParent();
620 const MachineRegisterInfo *MRI = &MF->getRegInfo();
621
622 auto GetMBBPressure =
623 [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
624 RegionPressure Pressure;
625 RegPressureTracker RPTracker(Pressure);
626
627 // Initialize the register pressure tracker.
628 RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
629 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
630
631 for (const auto &MI : reverse(*MBB)) {
632 if (MI.isDebugValue() || MI.isDebugLabel())
633 continue;
634 RegisterOperands RegOpers;
635 RegOpers.collect(MI, *TRI, *MRI, false, false);
636 RPTracker.recedeSkipDebugValues();
637 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
638 RPTracker.recede(RegOpers);
639 }
640
641 // Close the RPTracker to finalize live ins.
642 RPTracker.closeRegion();
643
644 return RPTracker.getPressure().MaxSetPressure;
645 };
646
647 // For now we only care about float and double type fma.
648 unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
649 *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
650
651 // Only reduce register pressure when pressure is high.
652 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
653 (float)VSSRCLimit * FMARPFactor;
654 }
655
isLoadFromConstantPool(MachineInstr * I) const656 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
657 // I has only one memory operand which is load from constant pool.
658 if (!I->hasOneMemOperand())
659 return false;
660
661 MachineMemOperand *Op = I->memoperands()[0];
662 return Op->isLoad() && Op->getPseudoValue() &&
663 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
664 }
665
generateLoadForNewConst(unsigned Idx,MachineInstr * MI,Type * Ty,SmallVectorImpl<MachineInstr * > & InsInstrs) const666 Register PPCInstrInfo::generateLoadForNewConst(
667 unsigned Idx, MachineInstr *MI, Type *Ty,
668 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
669 // Now we only support PPC64, Medium code model and P9 with vector.
670 // We have immutable pattern to access const pool. See function
671 // shouldReduceRegisterPressure.
672 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
673 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
674 "Target not supported!\n");
675
676 MachineFunction *MF = MI->getMF();
677 MachineRegisterInfo *MRI = &MF->getRegInfo();
678
679 // Generate ADDIStocHA8
680 Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
681 MachineInstrBuilder TOCOffset =
682 BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
683 .addReg(PPC::X2)
684 .addConstantPoolIndex(Idx);
685
686 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
687 "Only float and double are supported!");
688
689 unsigned LoadOpcode;
690 // Should be float type or double type.
691 if (Ty->isFloatTy())
692 LoadOpcode = PPC::DFLOADf32;
693 else
694 LoadOpcode = PPC::DFLOADf64;
695
696 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
697 Register VReg2 = MRI->createVirtualRegister(RC);
698 MachineMemOperand *MMO = MF->getMachineMemOperand(
699 MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad,
700 Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
701
702 // Generate Load from constant pool.
703 MachineInstrBuilder Load =
704 BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
705 .addConstantPoolIndex(Idx)
706 .addReg(VReg1, getKillRegState(true))
707 .addMemOperand(MMO);
708
709 Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
710
711 // Insert the toc load instructions into InsInstrs.
712 InsInstrs.insert(InsInstrs.begin(), Load);
713 InsInstrs.insert(InsInstrs.begin(), TOCOffset);
714 return VReg2;
715 }
716
717 // This function returns the const value in constant pool if the \p I is a load
718 // from constant pool.
719 const Constant *
getConstantFromConstantPool(MachineInstr * I) const720 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
721 MachineFunction *MF = I->getMF();
722 MachineRegisterInfo *MRI = &MF->getRegInfo();
723 MachineConstantPool *MCP = MF->getConstantPool();
724 assert(I->mayLoad() && "Should be a load instruction.\n");
725 for (auto MO : I->uses()) {
726 if (!MO.isReg())
727 continue;
728 Register Reg = MO.getReg();
729 if (Reg == 0 || !Reg.isVirtual())
730 continue;
731 // Find the toc address.
732 MachineInstr *DefMI = MRI->getVRegDef(Reg);
733 for (auto MO2 : DefMI->uses())
734 if (MO2.isCPI())
735 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
736 }
737 return nullptr;
738 }
739
getMachineCombinerPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns,bool DoRegPressureReduce) const740 bool PPCInstrInfo::getMachineCombinerPatterns(
741 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
742 bool DoRegPressureReduce) const {
743 // Using the machine combiner in this way is potentially expensive, so
744 // restrict to when aggressive optimizations are desired.
745 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOptLevel::Aggressive)
746 return false;
747
748 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
749 return true;
750
751 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
752 DoRegPressureReduce);
753 }
754
genAlternativeCodeSequence(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const755 void PPCInstrInfo::genAlternativeCodeSequence(
756 MachineInstr &Root, MachineCombinerPattern Pattern,
757 SmallVectorImpl<MachineInstr *> &InsInstrs,
758 SmallVectorImpl<MachineInstr *> &DelInstrs,
759 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
760 switch (Pattern) {
761 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
762 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
763 case MachineCombinerPattern::REASSOC_XY_BCA:
764 case MachineCombinerPattern::REASSOC_XY_BAC:
765 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
766 break;
767 default:
768 // Reassociate default patterns.
769 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
770 DelInstrs, InstrIdxForVirtReg);
771 break;
772 }
773 }
774
reassociateFMA(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const775 void PPCInstrInfo::reassociateFMA(
776 MachineInstr &Root, MachineCombinerPattern Pattern,
777 SmallVectorImpl<MachineInstr *> &InsInstrs,
778 SmallVectorImpl<MachineInstr *> &DelInstrs,
779 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
780 MachineFunction *MF = Root.getMF();
781 MachineRegisterInfo &MRI = MF->getRegInfo();
782 const TargetRegisterInfo *TRI = &getRegisterInfo();
783 MachineOperand &OpC = Root.getOperand(0);
784 Register RegC = OpC.getReg();
785 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
786 MRI.constrainRegClass(RegC, RC);
787
788 unsigned FmaOp = Root.getOpcode();
789 int16_t Idx = getFMAOpIdxInfo(FmaOp);
790 assert(Idx >= 0 && "Root must be a FMA instruction");
791
792 bool IsILPReassociate =
793 (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
794 (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
795
796 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
797 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
798
799 MachineInstr *Prev = nullptr;
800 MachineInstr *Leaf = nullptr;
801 switch (Pattern) {
802 default:
803 llvm_unreachable("not recognized pattern!");
804 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
805 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
806 Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
807 Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
808 break;
809 case MachineCombinerPattern::REASSOC_XY_BAC: {
810 Register MULReg =
811 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
812 Leaf = MRI.getVRegDef(MULReg);
813 break;
814 }
815 case MachineCombinerPattern::REASSOC_XY_BCA: {
816 Register MULReg = TRI->lookThruCopyLike(
817 Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
818 Leaf = MRI.getVRegDef(MULReg);
819 break;
820 }
821 }
822
823 uint32_t IntersectedFlags = 0;
824 if (IsILPReassociate)
825 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
826 else
827 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
828
829 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
830 bool &KillFlag) {
831 Reg = Operand.getReg();
832 MRI.constrainRegClass(Reg, RC);
833 KillFlag = Operand.isKill();
834 };
835
836 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
837 Register &MulOp2, Register &AddOp,
838 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
839 bool &AddOpKillFlag) {
840 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
841 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
842 GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
843 };
844
845 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
846 RegA21, RegB;
847 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
848 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
849 KillA11 = false, KillA21 = false, KillB = false;
850
851 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
852
853 if (IsILPReassociate)
854 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
855
856 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
857 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
858 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
859 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
860 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
861 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
862 } else {
863 // Get FSUB instruction info.
864 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
865 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
866 }
867
868 // Create new virtual registers for the new results instead of
869 // recycling legacy ones because the MachineCombiner's computation of the
870 // critical path requires a new register definition rather than an existing
871 // one.
872 // For register pressure reassociation, we only need create one virtual
873 // register for the new fma.
874 Register NewVRA = MRI.createVirtualRegister(RC);
875 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
876
877 Register NewVRB = 0;
878 if (IsILPReassociate) {
879 NewVRB = MRI.createVirtualRegister(RC);
880 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
881 }
882
883 Register NewVRD = 0;
884 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
885 NewVRD = MRI.createVirtualRegister(RC);
886 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
887 }
888
889 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
890 Register RegMul1, bool KillRegMul1,
891 Register RegMul2, bool KillRegMul2) {
892 MI->getOperand(AddOpIdx).setReg(RegAdd);
893 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
894 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
895 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
896 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
897 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
898 };
899
900 MachineInstrBuilder NewARegPressure, NewCRegPressure;
901 switch (Pattern) {
902 default:
903 llvm_unreachable("not recognized pattern!");
904 case MachineCombinerPattern::REASSOC_XY_AMM_BMM: {
905 // Create new instructions for insertion.
906 MachineInstrBuilder MINewB =
907 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
908 .addReg(RegX, getKillRegState(KillX))
909 .addReg(RegM21, getKillRegState(KillM21))
910 .addReg(RegM22, getKillRegState(KillM22));
911 MachineInstrBuilder MINewA =
912 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
913 .addReg(RegY, getKillRegState(KillY))
914 .addReg(RegM31, getKillRegState(KillM31))
915 .addReg(RegM32, getKillRegState(KillM32));
916 // If AddOpIdx is not 1, adjust the order.
917 if (AddOpIdx != 1) {
918 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
919 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
920 }
921
922 MachineInstrBuilder MINewC =
923 BuildMI(*MF, Root.getDebugLoc(),
924 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
925 .addReg(NewVRB, getKillRegState(true))
926 .addReg(NewVRA, getKillRegState(true));
927
928 // Update flags for newly created instructions.
929 setSpecialOperandAttr(*MINewA, IntersectedFlags);
930 setSpecialOperandAttr(*MINewB, IntersectedFlags);
931 setSpecialOperandAttr(*MINewC, IntersectedFlags);
932
933 // Record new instructions for insertion.
934 InsInstrs.push_back(MINewA);
935 InsInstrs.push_back(MINewB);
936 InsInstrs.push_back(MINewC);
937 break;
938 }
939 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
940 assert(NewVRD && "new FMA register not created!");
941 // Create new instructions for insertion.
942 MachineInstrBuilder MINewA =
943 BuildMI(*MF, Leaf->getDebugLoc(),
944 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
945 .addReg(RegM11, getKillRegState(KillM11))
946 .addReg(RegM12, getKillRegState(KillM12));
947 MachineInstrBuilder MINewB =
948 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
949 .addReg(RegX, getKillRegState(KillX))
950 .addReg(RegM21, getKillRegState(KillM21))
951 .addReg(RegM22, getKillRegState(KillM22));
952 MachineInstrBuilder MINewD =
953 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
954 .addReg(NewVRA, getKillRegState(true))
955 .addReg(RegM31, getKillRegState(KillM31))
956 .addReg(RegM32, getKillRegState(KillM32));
957 // If AddOpIdx is not 1, adjust the order.
958 if (AddOpIdx != 1) {
959 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
960 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
961 KillM32);
962 }
963
964 MachineInstrBuilder MINewC =
965 BuildMI(*MF, Root.getDebugLoc(),
966 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
967 .addReg(NewVRB, getKillRegState(true))
968 .addReg(NewVRD, getKillRegState(true));
969
970 // Update flags for newly created instructions.
971 setSpecialOperandAttr(*MINewA, IntersectedFlags);
972 setSpecialOperandAttr(*MINewB, IntersectedFlags);
973 setSpecialOperandAttr(*MINewD, IntersectedFlags);
974 setSpecialOperandAttr(*MINewC, IntersectedFlags);
975
976 // Record new instructions for insertion.
977 InsInstrs.push_back(MINewA);
978 InsInstrs.push_back(MINewB);
979 InsInstrs.push_back(MINewD);
980 InsInstrs.push_back(MINewC);
981 break;
982 }
983 case MachineCombinerPattern::REASSOC_XY_BAC:
984 case MachineCombinerPattern::REASSOC_XY_BCA: {
985 Register VarReg;
986 bool KillVarReg = false;
987 if (Pattern == MachineCombinerPattern::REASSOC_XY_BCA) {
988 VarReg = RegM31;
989 KillVarReg = KillM31;
990 } else {
991 VarReg = RegM32;
992 KillVarReg = KillM32;
993 }
994 // We don't want to get negative const from memory pool too early, as the
995 // created entry will not be deleted even if it has no users. Since all
996 // operand of Leaf and Root are virtual register, we use zero register
997 // here as a placeholder. When the InsInstrs is selected in
998 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
999 // with a virtual register which is a load from constant pool.
1000 NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1001 .addReg(RegB, getKillRegState(RegB))
1002 .addReg(RegY, getKillRegState(KillY))
1003 .addReg(PPC::ZERO8);
1004 NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1005 .addReg(NewVRA, getKillRegState(true))
1006 .addReg(RegX, getKillRegState(KillX))
1007 .addReg(VarReg, getKillRegState(KillVarReg));
1008 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1009 // both at index 1, no need to adjust.
1010 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1011 // the operand index here.
1012 break;
1013 }
1014 }
1015
1016 if (!IsILPReassociate) {
1017 setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1018 setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1019
1020 InsInstrs.push_back(NewARegPressure);
1021 InsInstrs.push_back(NewCRegPressure);
1022 }
1023
1024 assert(!InsInstrs.empty() &&
1025 "Insertion instructions set should not be empty!");
1026
1027 // Record old instructions for deletion.
1028 DelInstrs.push_back(Leaf);
1029 if (IsILPReassociate)
1030 DelInstrs.push_back(Prev);
1031 DelInstrs.push_back(&Root);
1032 }
1033
1034 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
isCoalescableExtInstr(const MachineInstr & MI,Register & SrcReg,Register & DstReg,unsigned & SubIdx) const1035 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1036 Register &SrcReg, Register &DstReg,
1037 unsigned &SubIdx) const {
1038 switch (MI.getOpcode()) {
1039 default: return false;
1040 case PPC::EXTSW:
1041 case PPC::EXTSW_32:
1042 case PPC::EXTSW_32_64:
1043 SrcReg = MI.getOperand(1).getReg();
1044 DstReg = MI.getOperand(0).getReg();
1045 SubIdx = PPC::sub_32;
1046 return true;
1047 }
1048 }
1049
isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const1050 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1051 int &FrameIndex) const {
1052 if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
1053 // Check for the operands added by addFrameReference (the immediate is the
1054 // offset which defaults to 0).
1055 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1056 MI.getOperand(2).isFI()) {
1057 FrameIndex = MI.getOperand(2).getIndex();
1058 return MI.getOperand(0).getReg();
1059 }
1060 }
1061 return 0;
1062 }
1063
1064 // For opcodes with the ReMaterializable flag set, this function is called to
1065 // verify the instruction is really rematable.
isReallyTriviallyReMaterializable(const MachineInstr & MI) const1066 bool PPCInstrInfo::isReallyTriviallyReMaterializable(
1067 const MachineInstr &MI) const {
1068 switch (MI.getOpcode()) {
1069 default:
1070 // Let base implementaion decide.
1071 break;
1072 case PPC::LI:
1073 case PPC::LI8:
1074 case PPC::PLI:
1075 case PPC::PLI8:
1076 case PPC::LIS:
1077 case PPC::LIS8:
1078 case PPC::ADDIStocHA:
1079 case PPC::ADDIStocHA8:
1080 case PPC::ADDItocL:
1081 case PPC::LOAD_STACK_GUARD:
1082 case PPC::XXLXORz:
1083 case PPC::XXLXORspz:
1084 case PPC::XXLXORdpz:
1085 case PPC::XXLEQVOnes:
1086 case PPC::XXSPLTI32DX:
1087 case PPC::XXSPLTIW:
1088 case PPC::XXSPLTIDP:
1089 case PPC::V_SET0B:
1090 case PPC::V_SET0H:
1091 case PPC::V_SET0:
1092 case PPC::V_SETALLONESB:
1093 case PPC::V_SETALLONESH:
1094 case PPC::V_SETALLONES:
1095 case PPC::CRSET:
1096 case PPC::CRUNSET:
1097 case PPC::XXSETACCZ:
1098 case PPC::XXSETACCZW:
1099 return true;
1100 }
1101 return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
1102 }
1103
isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const1104 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1105 int &FrameIndex) const {
1106 if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
1107 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1108 MI.getOperand(2).isFI()) {
1109 FrameIndex = MI.getOperand(2).getIndex();
1110 return MI.getOperand(0).getReg();
1111 }
1112 }
1113 return 0;
1114 }
1115
commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const1116 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1117 unsigned OpIdx1,
1118 unsigned OpIdx2) const {
1119 MachineFunction &MF = *MI.getParent()->getParent();
1120
1121 // Normal instructions can be commuted the obvious way.
1122 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1123 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1124 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1125 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1126 // changing the relative order of the mask operands might change what happens
1127 // to the high-bits of the mask (and, thus, the result).
1128
1129 // Cannot commute if it has a non-zero rotate count.
1130 if (MI.getOperand(3).getImm() != 0)
1131 return nullptr;
1132
1133 // If we have a zero rotate count, we have:
1134 // M = mask(MB,ME)
1135 // Op0 = (Op1 & ~M) | (Op2 & M)
1136 // Change this to:
1137 // M = mask((ME+1)&31, (MB-1)&31)
1138 // Op0 = (Op2 & ~M) | (Op1 & M)
1139
1140 // Swap op1/op2
1141 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1142 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1143 Register Reg0 = MI.getOperand(0).getReg();
1144 Register Reg1 = MI.getOperand(1).getReg();
1145 Register Reg2 = MI.getOperand(2).getReg();
1146 unsigned SubReg1 = MI.getOperand(1).getSubReg();
1147 unsigned SubReg2 = MI.getOperand(2).getSubReg();
1148 bool Reg1IsKill = MI.getOperand(1).isKill();
1149 bool Reg2IsKill = MI.getOperand(2).isKill();
1150 bool ChangeReg0 = false;
1151 // If machine instrs are no longer in two-address forms, update
1152 // destination register as well.
1153 if (Reg0 == Reg1) {
1154 // Must be two address instruction (i.e. op1 is tied to op0).
1155 assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
1156 "Expecting a two-address instruction!");
1157 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1158 Reg2IsKill = false;
1159 ChangeReg0 = true;
1160 }
1161
1162 // Masks.
1163 unsigned MB = MI.getOperand(4).getImm();
1164 unsigned ME = MI.getOperand(5).getImm();
1165
1166 // We can't commute a trivial mask (there is no way to represent an all-zero
1167 // mask).
1168 if (MB == 0 && ME == 31)
1169 return nullptr;
1170
1171 if (NewMI) {
1172 // Create a new instruction.
1173 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1174 bool Reg0IsDead = MI.getOperand(0).isDead();
1175 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1176 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1177 .addReg(Reg2, getKillRegState(Reg2IsKill))
1178 .addReg(Reg1, getKillRegState(Reg1IsKill))
1179 .addImm((ME + 1) & 31)
1180 .addImm((MB - 1) & 31);
1181 }
1182
1183 if (ChangeReg0) {
1184 MI.getOperand(0).setReg(Reg2);
1185 MI.getOperand(0).setSubReg(SubReg2);
1186 }
1187 MI.getOperand(2).setReg(Reg1);
1188 MI.getOperand(1).setReg(Reg2);
1189 MI.getOperand(2).setSubReg(SubReg1);
1190 MI.getOperand(1).setSubReg(SubReg2);
1191 MI.getOperand(2).setIsKill(Reg1IsKill);
1192 MI.getOperand(1).setIsKill(Reg2IsKill);
1193
1194 // Swap the mask around.
1195 MI.getOperand(4).setImm((ME + 1) & 31);
1196 MI.getOperand(5).setImm((MB - 1) & 31);
1197 return &MI;
1198 }
1199
findCommutedOpIndices(const MachineInstr & MI,unsigned & SrcOpIdx1,unsigned & SrcOpIdx2) const1200 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1201 unsigned &SrcOpIdx1,
1202 unsigned &SrcOpIdx2) const {
1203 // For VSX A-Type FMA instructions, it is the first two operands that can be
1204 // commuted, however, because the non-encoded tied input operand is listed
1205 // first, the operands to swap are actually the second and third.
1206
1207 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1208 if (AltOpc == -1)
1209 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1210
1211 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1212 // and SrcOpIdx2.
1213 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1214 }
1215
insertNoop(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const1216 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
1217 MachineBasicBlock::iterator MI) const {
1218 // This function is used for scheduling, and the nop wanted here is the type
1219 // that terminates dispatch groups on the POWER cores.
1220 unsigned Directive = Subtarget.getCPUDirective();
1221 unsigned Opcode;
1222 switch (Directive) {
1223 default: Opcode = PPC::NOP; break;
1224 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1225 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1226 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1227 // FIXME: Update when POWER9 scheduling model is ready.
1228 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1229 }
1230
1231 DebugLoc DL;
1232 BuildMI(MBB, MI, DL, get(Opcode));
1233 }
1234
1235 /// Return the noop instruction to use for a noop.
getNop() const1236 MCInst PPCInstrInfo::getNop() const {
1237 MCInst Nop;
1238 Nop.setOpcode(PPC::NOP);
1239 return Nop;
1240 }
1241
1242 // Branch analysis.
1243 // Note: If the condition register is set to CTR or CTR8 then this is a
1244 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const1245 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1246 MachineBasicBlock *&TBB,
1247 MachineBasicBlock *&FBB,
1248 SmallVectorImpl<MachineOperand> &Cond,
1249 bool AllowModify) const {
1250 bool isPPC64 = Subtarget.isPPC64();
1251
1252 // If the block has no terminators, it just falls into the block after it.
1253 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1254 if (I == MBB.end())
1255 return false;
1256
1257 if (!isUnpredicatedTerminator(*I))
1258 return false;
1259
1260 if (AllowModify) {
1261 // If the BB ends with an unconditional branch to the fallthrough BB,
1262 // we eliminate the branch instruction.
1263 if (I->getOpcode() == PPC::B &&
1264 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1265 I->eraseFromParent();
1266
1267 // We update iterator after deleting the last branch.
1268 I = MBB.getLastNonDebugInstr();
1269 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1270 return false;
1271 }
1272 }
1273
1274 // Get the last instruction in the block.
1275 MachineInstr &LastInst = *I;
1276
1277 // If there is only one terminator instruction, process it.
1278 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1279 if (LastInst.getOpcode() == PPC::B) {
1280 if (!LastInst.getOperand(0).isMBB())
1281 return true;
1282 TBB = LastInst.getOperand(0).getMBB();
1283 return false;
1284 } else if (LastInst.getOpcode() == PPC::BCC) {
1285 if (!LastInst.getOperand(2).isMBB())
1286 return true;
1287 // Block ends with fall-through condbranch.
1288 TBB = LastInst.getOperand(2).getMBB();
1289 Cond.push_back(LastInst.getOperand(0));
1290 Cond.push_back(LastInst.getOperand(1));
1291 return false;
1292 } else if (LastInst.getOpcode() == PPC::BC) {
1293 if (!LastInst.getOperand(1).isMBB())
1294 return true;
1295 // Block ends with fall-through condbranch.
1296 TBB = LastInst.getOperand(1).getMBB();
1297 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1298 Cond.push_back(LastInst.getOperand(0));
1299 return false;
1300 } else if (LastInst.getOpcode() == PPC::BCn) {
1301 if (!LastInst.getOperand(1).isMBB())
1302 return true;
1303 // Block ends with fall-through condbranch.
1304 TBB = LastInst.getOperand(1).getMBB();
1305 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1306 Cond.push_back(LastInst.getOperand(0));
1307 return false;
1308 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1309 LastInst.getOpcode() == PPC::BDNZ) {
1310 if (!LastInst.getOperand(0).isMBB())
1311 return true;
1312 if (DisableCTRLoopAnal)
1313 return true;
1314 TBB = LastInst.getOperand(0).getMBB();
1315 Cond.push_back(MachineOperand::CreateImm(1));
1316 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1317 true));
1318 return false;
1319 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1320 LastInst.getOpcode() == PPC::BDZ) {
1321 if (!LastInst.getOperand(0).isMBB())
1322 return true;
1323 if (DisableCTRLoopAnal)
1324 return true;
1325 TBB = LastInst.getOperand(0).getMBB();
1326 Cond.push_back(MachineOperand::CreateImm(0));
1327 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1328 true));
1329 return false;
1330 }
1331
1332 // Otherwise, don't know what this is.
1333 return true;
1334 }
1335
1336 // Get the instruction before it if it's a terminator.
1337 MachineInstr &SecondLastInst = *I;
1338
1339 // If there are three terminators, we don't know what sort of block this is.
1340 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1341 return true;
1342
1343 // If the block ends with PPC::B and PPC:BCC, handle it.
1344 if (SecondLastInst.getOpcode() == PPC::BCC &&
1345 LastInst.getOpcode() == PPC::B) {
1346 if (!SecondLastInst.getOperand(2).isMBB() ||
1347 !LastInst.getOperand(0).isMBB())
1348 return true;
1349 TBB = SecondLastInst.getOperand(2).getMBB();
1350 Cond.push_back(SecondLastInst.getOperand(0));
1351 Cond.push_back(SecondLastInst.getOperand(1));
1352 FBB = LastInst.getOperand(0).getMBB();
1353 return false;
1354 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1355 LastInst.getOpcode() == PPC::B) {
1356 if (!SecondLastInst.getOperand(1).isMBB() ||
1357 !LastInst.getOperand(0).isMBB())
1358 return true;
1359 TBB = SecondLastInst.getOperand(1).getMBB();
1360 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1361 Cond.push_back(SecondLastInst.getOperand(0));
1362 FBB = LastInst.getOperand(0).getMBB();
1363 return false;
1364 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1365 LastInst.getOpcode() == PPC::B) {
1366 if (!SecondLastInst.getOperand(1).isMBB() ||
1367 !LastInst.getOperand(0).isMBB())
1368 return true;
1369 TBB = SecondLastInst.getOperand(1).getMBB();
1370 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1371 Cond.push_back(SecondLastInst.getOperand(0));
1372 FBB = LastInst.getOperand(0).getMBB();
1373 return false;
1374 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1375 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1376 LastInst.getOpcode() == PPC::B) {
1377 if (!SecondLastInst.getOperand(0).isMBB() ||
1378 !LastInst.getOperand(0).isMBB())
1379 return true;
1380 if (DisableCTRLoopAnal)
1381 return true;
1382 TBB = SecondLastInst.getOperand(0).getMBB();
1383 Cond.push_back(MachineOperand::CreateImm(1));
1384 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1385 true));
1386 FBB = LastInst.getOperand(0).getMBB();
1387 return false;
1388 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1389 SecondLastInst.getOpcode() == PPC::BDZ) &&
1390 LastInst.getOpcode() == PPC::B) {
1391 if (!SecondLastInst.getOperand(0).isMBB() ||
1392 !LastInst.getOperand(0).isMBB())
1393 return true;
1394 if (DisableCTRLoopAnal)
1395 return true;
1396 TBB = SecondLastInst.getOperand(0).getMBB();
1397 Cond.push_back(MachineOperand::CreateImm(0));
1398 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1399 true));
1400 FBB = LastInst.getOperand(0).getMBB();
1401 return false;
1402 }
1403
1404 // If the block ends with two PPC:Bs, handle it. The second one is not
1405 // executed, so remove it.
1406 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1407 if (!SecondLastInst.getOperand(0).isMBB())
1408 return true;
1409 TBB = SecondLastInst.getOperand(0).getMBB();
1410 I = LastInst;
1411 if (AllowModify)
1412 I->eraseFromParent();
1413 return false;
1414 }
1415
1416 // Otherwise, can't handle this.
1417 return true;
1418 }
1419
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const1420 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1421 int *BytesRemoved) const {
1422 assert(!BytesRemoved && "code size not handled");
1423
1424 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1425 if (I == MBB.end())
1426 return 0;
1427
1428 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1429 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1430 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1431 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1432 return 0;
1433
1434 // Remove the branch.
1435 I->eraseFromParent();
1436
1437 I = MBB.end();
1438
1439 if (I == MBB.begin()) return 1;
1440 --I;
1441 if (I->getOpcode() != PPC::BCC &&
1442 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1443 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1444 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1445 return 1;
1446
1447 // Remove the branch.
1448 I->eraseFromParent();
1449 return 2;
1450 }
1451
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const1452 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1453 MachineBasicBlock *TBB,
1454 MachineBasicBlock *FBB,
1455 ArrayRef<MachineOperand> Cond,
1456 const DebugLoc &DL,
1457 int *BytesAdded) const {
1458 // Shouldn't be a fall through.
1459 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1460 assert((Cond.size() == 2 || Cond.size() == 0) &&
1461 "PPC branch conditions have two components!");
1462 assert(!BytesAdded && "code size not handled");
1463
1464 bool isPPC64 = Subtarget.isPPC64();
1465
1466 // One-way branch.
1467 if (!FBB) {
1468 if (Cond.empty()) // Unconditional branch
1469 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1470 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1471 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1472 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1473 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1474 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1475 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1476 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1477 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1478 else // Conditional branch
1479 BuildMI(&MBB, DL, get(PPC::BCC))
1480 .addImm(Cond[0].getImm())
1481 .add(Cond[1])
1482 .addMBB(TBB);
1483 return 1;
1484 }
1485
1486 // Two-way Conditional Branch.
1487 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1488 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1489 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1490 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1491 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1492 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1493 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1494 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1495 else
1496 BuildMI(&MBB, DL, get(PPC::BCC))
1497 .addImm(Cond[0].getImm())
1498 .add(Cond[1])
1499 .addMBB(TBB);
1500 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1501 return 2;
1502 }
1503
1504 // Select analysis.
canInsertSelect(const MachineBasicBlock & MBB,ArrayRef<MachineOperand> Cond,Register DstReg,Register TrueReg,Register FalseReg,int & CondCycles,int & TrueCycles,int & FalseCycles) const1505 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1506 ArrayRef<MachineOperand> Cond,
1507 Register DstReg, Register TrueReg,
1508 Register FalseReg, int &CondCycles,
1509 int &TrueCycles, int &FalseCycles) const {
1510 if (!Subtarget.hasISEL())
1511 return false;
1512
1513 if (Cond.size() != 2)
1514 return false;
1515
1516 // If this is really a bdnz-like condition, then it cannot be turned into a
1517 // select.
1518 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1519 return false;
1520
1521 // If the conditional branch uses a physical register, then it cannot be
1522 // turned into a select.
1523 if (Cond[1].getReg().isPhysical())
1524 return false;
1525
1526 // Check register classes.
1527 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1528 const TargetRegisterClass *RC =
1529 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1530 if (!RC)
1531 return false;
1532
1533 // isel is for regular integer GPRs only.
1534 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1535 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1536 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1537 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1538 return false;
1539
1540 // FIXME: These numbers are for the A2, how well they work for other cores is
1541 // an open question. On the A2, the isel instruction has a 2-cycle latency
1542 // but single-cycle throughput. These numbers are used in combination with
1543 // the MispredictPenalty setting from the active SchedMachineModel.
1544 CondCycles = 1;
1545 TrueCycles = 1;
1546 FalseCycles = 1;
1547
1548 return true;
1549 }
1550
insertSelect(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & dl,Register DestReg,ArrayRef<MachineOperand> Cond,Register TrueReg,Register FalseReg) const1551 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1552 MachineBasicBlock::iterator MI,
1553 const DebugLoc &dl, Register DestReg,
1554 ArrayRef<MachineOperand> Cond, Register TrueReg,
1555 Register FalseReg) const {
1556 assert(Cond.size() == 2 &&
1557 "PPC branch conditions have two components!");
1558
1559 // Get the register classes.
1560 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1561 const TargetRegisterClass *RC =
1562 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1563 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1564
1565 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1566 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1567 assert((Is64Bit ||
1568 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1569 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1570 "isel is for regular integer GPRs only");
1571
1572 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1573 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1574
1575 unsigned SubIdx = 0;
1576 bool SwapOps = false;
1577 switch (SelectPred) {
1578 case PPC::PRED_EQ:
1579 case PPC::PRED_EQ_MINUS:
1580 case PPC::PRED_EQ_PLUS:
1581 SubIdx = PPC::sub_eq; SwapOps = false; break;
1582 case PPC::PRED_NE:
1583 case PPC::PRED_NE_MINUS:
1584 case PPC::PRED_NE_PLUS:
1585 SubIdx = PPC::sub_eq; SwapOps = true; break;
1586 case PPC::PRED_LT:
1587 case PPC::PRED_LT_MINUS:
1588 case PPC::PRED_LT_PLUS:
1589 SubIdx = PPC::sub_lt; SwapOps = false; break;
1590 case PPC::PRED_GE:
1591 case PPC::PRED_GE_MINUS:
1592 case PPC::PRED_GE_PLUS:
1593 SubIdx = PPC::sub_lt; SwapOps = true; break;
1594 case PPC::PRED_GT:
1595 case PPC::PRED_GT_MINUS:
1596 case PPC::PRED_GT_PLUS:
1597 SubIdx = PPC::sub_gt; SwapOps = false; break;
1598 case PPC::PRED_LE:
1599 case PPC::PRED_LE_MINUS:
1600 case PPC::PRED_LE_PLUS:
1601 SubIdx = PPC::sub_gt; SwapOps = true; break;
1602 case PPC::PRED_UN:
1603 case PPC::PRED_UN_MINUS:
1604 case PPC::PRED_UN_PLUS:
1605 SubIdx = PPC::sub_un; SwapOps = false; break;
1606 case PPC::PRED_NU:
1607 case PPC::PRED_NU_MINUS:
1608 case PPC::PRED_NU_PLUS:
1609 SubIdx = PPC::sub_un; SwapOps = true; break;
1610 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1611 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1612 }
1613
1614 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1615 SecondReg = SwapOps ? TrueReg : FalseReg;
1616
1617 // The first input register of isel cannot be r0. If it is a member
1618 // of a register class that can be r0, then copy it first (the
1619 // register allocator should eliminate the copy).
1620 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1621 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1622 const TargetRegisterClass *FirstRC =
1623 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1624 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1625 Register OldFirstReg = FirstReg;
1626 FirstReg = MRI.createVirtualRegister(FirstRC);
1627 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1628 .addReg(OldFirstReg);
1629 }
1630
1631 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1632 .addReg(FirstReg).addReg(SecondReg)
1633 .addReg(Cond[1].getReg(), 0, SubIdx);
1634 }
1635
getCRBitValue(unsigned CRBit)1636 static unsigned getCRBitValue(unsigned CRBit) {
1637 unsigned Ret = 4;
1638 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1639 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1640 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1641 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1642 Ret = 3;
1643 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1644 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1645 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1646 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1647 Ret = 2;
1648 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1649 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1650 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1651 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1652 Ret = 1;
1653 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1654 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1655 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1656 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1657 Ret = 0;
1658
1659 assert(Ret != 4 && "Invalid CR bit register");
1660 return Ret;
1661 }
1662
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const1663 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1664 MachineBasicBlock::iterator I,
1665 const DebugLoc &DL, MCRegister DestReg,
1666 MCRegister SrcReg, bool KillSrc) const {
1667 // We can end up with self copies and similar things as a result of VSX copy
1668 // legalization. Promote them here.
1669 const TargetRegisterInfo *TRI = &getRegisterInfo();
1670 if (PPC::F8RCRegClass.contains(DestReg) &&
1671 PPC::VSRCRegClass.contains(SrcReg)) {
1672 MCRegister SuperReg =
1673 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1674
1675 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1676 llvm_unreachable("nop VSX copy");
1677
1678 DestReg = SuperReg;
1679 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1680 PPC::VSRCRegClass.contains(DestReg)) {
1681 MCRegister SuperReg =
1682 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1683
1684 if (VSXSelfCopyCrash && DestReg == SuperReg)
1685 llvm_unreachable("nop VSX copy");
1686
1687 SrcReg = SuperReg;
1688 }
1689
1690 // Different class register copy
1691 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1692 PPC::GPRCRegClass.contains(DestReg)) {
1693 MCRegister CRReg = getCRFromCRBit(SrcReg);
1694 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1695 getKillRegState(KillSrc);
1696 // Rotate the CR bit in the CR fields to be the least significant bit and
1697 // then mask with 0x1 (MB = ME = 31).
1698 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1699 .addReg(DestReg, RegState::Kill)
1700 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1701 .addImm(31)
1702 .addImm(31);
1703 return;
1704 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1705 (PPC::G8RCRegClass.contains(DestReg) ||
1706 PPC::GPRCRegClass.contains(DestReg))) {
1707 bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1708 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1709 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1710 unsigned CRNum = TRI->getEncodingValue(SrcReg);
1711 BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1712 getKillRegState(KillSrc);
1713 if (CRNum == 7)
1714 return;
1715 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1716 BuildMI(MBB, I, DL, get(ShCode), DestReg)
1717 .addReg(DestReg, RegState::Kill)
1718 .addImm(CRNum * 4 + 4)
1719 .addImm(28)
1720 .addImm(31);
1721 return;
1722 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1723 PPC::VSFRCRegClass.contains(DestReg)) {
1724 assert(Subtarget.hasDirectMove() &&
1725 "Subtarget doesn't support directmove, don't know how to copy.");
1726 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1727 NumGPRtoVSRSpill++;
1728 getKillRegState(KillSrc);
1729 return;
1730 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1731 PPC::G8RCRegClass.contains(DestReg)) {
1732 assert(Subtarget.hasDirectMove() &&
1733 "Subtarget doesn't support directmove, don't know how to copy.");
1734 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1735 getKillRegState(KillSrc);
1736 return;
1737 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1738 PPC::GPRCRegClass.contains(DestReg)) {
1739 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1740 getKillRegState(KillSrc);
1741 return;
1742 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1743 PPC::SPERCRegClass.contains(DestReg)) {
1744 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1745 getKillRegState(KillSrc);
1746 return;
1747 }
1748
1749 unsigned Opc;
1750 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1751 Opc = PPC::OR;
1752 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1753 Opc = PPC::OR8;
1754 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1755 Opc = PPC::FMR;
1756 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1757 Opc = PPC::MCRF;
1758 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1759 Opc = PPC::VOR;
1760 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1761 // There are two different ways this can be done:
1762 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1763 // issue in VSU pipeline 0.
1764 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1765 // can go to either pipeline.
1766 // We'll always use xxlor here, because in practically all cases where
1767 // copies are generated, they are close enough to some use that the
1768 // lower-latency form is preferable.
1769 Opc = PPC::XXLOR;
1770 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1771 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1772 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1773 else if (Subtarget.pairedVectorMemops() &&
1774 PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1775 if (SrcReg > PPC::VSRp15)
1776 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1777 else
1778 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1779 if (DestReg > PPC::VSRp15)
1780 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1781 else
1782 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1783 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1784 addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1785 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1786 addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1787 return;
1788 }
1789 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1790 Opc = PPC::CROR;
1791 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1792 Opc = PPC::EVOR;
1793 else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1794 PPC::UACCRCRegClass.contains(DestReg)) &&
1795 (PPC::ACCRCRegClass.contains(SrcReg) ||
1796 PPC::UACCRCRegClass.contains(SrcReg))) {
1797 // If primed, de-prime the source register, copy the individual registers
1798 // and prime the destination if needed. The vector subregisters are
1799 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1800 // source is primed, we need to re-prime it after the copy as well.
1801 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1802 bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1803 bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1804 MCRegister VSLSrcReg =
1805 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1806 MCRegister VSLDestReg =
1807 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1808 if (SrcPrimed)
1809 BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1810 for (unsigned Idx = 0; Idx < 4; Idx++)
1811 BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1812 .addReg(VSLSrcReg + Idx)
1813 .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1814 if (DestPrimed)
1815 BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1816 if (SrcPrimed && !KillSrc)
1817 BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1818 return;
1819 } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1820 PPC::G8pRCRegClass.contains(SrcReg)) {
1821 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1822 unsigned DestRegIdx = DestReg - PPC::G8p0;
1823 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1824 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1825 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1826 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1827 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1828 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1829 .addReg(SrcRegSub0)
1830 .addReg(SrcRegSub0, getKillRegState(KillSrc));
1831 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1832 .addReg(SrcRegSub1)
1833 .addReg(SrcRegSub1, getKillRegState(KillSrc));
1834 return;
1835 } else
1836 llvm_unreachable("Impossible reg-to-reg copy");
1837
1838 const MCInstrDesc &MCID = get(Opc);
1839 if (MCID.getNumOperands() == 3)
1840 BuildMI(MBB, I, DL, MCID, DestReg)
1841 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1842 else
1843 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1844 }
1845
getSpillIndex(const TargetRegisterClass * RC) const1846 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1847 int OpcodeIndex = 0;
1848
1849 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1850 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1851 OpcodeIndex = SOK_Int4Spill;
1852 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1853 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1854 OpcodeIndex = SOK_Int8Spill;
1855 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1856 OpcodeIndex = SOK_Float8Spill;
1857 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1858 OpcodeIndex = SOK_Float4Spill;
1859 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1860 OpcodeIndex = SOK_SPESpill;
1861 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1862 OpcodeIndex = SOK_CRSpill;
1863 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1864 OpcodeIndex = SOK_CRBitSpill;
1865 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1866 OpcodeIndex = SOK_VRVectorSpill;
1867 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1868 OpcodeIndex = SOK_VSXVectorSpill;
1869 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1870 OpcodeIndex = SOK_VectorFloat8Spill;
1871 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1872 OpcodeIndex = SOK_VectorFloat4Spill;
1873 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1874 OpcodeIndex = SOK_SpillToVSR;
1875 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1876 assert(Subtarget.pairedVectorMemops() &&
1877 "Register unexpected when paired memops are disabled.");
1878 OpcodeIndex = SOK_AccumulatorSpill;
1879 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1880 assert(Subtarget.pairedVectorMemops() &&
1881 "Register unexpected when paired memops are disabled.");
1882 OpcodeIndex = SOK_UAccumulatorSpill;
1883 } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1884 assert(Subtarget.pairedVectorMemops() &&
1885 "Register unexpected when paired memops are disabled.");
1886 OpcodeIndex = SOK_WAccumulatorSpill;
1887 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1888 assert(Subtarget.pairedVectorMemops() &&
1889 "Register unexpected when paired memops are disabled.");
1890 OpcodeIndex = SOK_PairedVecSpill;
1891 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1892 OpcodeIndex = SOK_PairedG8Spill;
1893 } else {
1894 llvm_unreachable("Unknown regclass!");
1895 }
1896 return OpcodeIndex;
1897 }
1898
1899 unsigned
getStoreOpcodeForSpill(const TargetRegisterClass * RC) const1900 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1901 ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1902 return OpcodesForSpill[getSpillIndex(RC)];
1903 }
1904
1905 unsigned
getLoadOpcodeForSpill(const TargetRegisterClass * RC) const1906 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1907 ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1908 return OpcodesForSpill[getSpillIndex(RC)];
1909 }
1910
StoreRegToStackSlot(MachineFunction & MF,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1911 void PPCInstrInfo::StoreRegToStackSlot(
1912 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1913 const TargetRegisterClass *RC,
1914 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1915 unsigned Opcode = getStoreOpcodeForSpill(RC);
1916 DebugLoc DL;
1917
1918 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1919 FuncInfo->setHasSpills();
1920
1921 NewMIs.push_back(addFrameReference(
1922 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1923 FrameIdx));
1924
1925 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1926 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1927 FuncInfo->setSpillsCR();
1928
1929 if (isXFormMemOp(Opcode))
1930 FuncInfo->setHasNonRISpills();
1931 }
1932
storeRegToStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1933 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1934 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1935 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1936 const TargetRegisterInfo *TRI) const {
1937 MachineFunction &MF = *MBB.getParent();
1938 SmallVector<MachineInstr *, 4> NewMIs;
1939
1940 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1941
1942 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1943 MBB.insert(MI, NewMIs[i]);
1944
1945 const MachineFrameInfo &MFI = MF.getFrameInfo();
1946 MachineMemOperand *MMO = MF.getMachineMemOperand(
1947 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1948 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1949 MFI.getObjectAlign(FrameIdx));
1950 NewMIs.back()->addMemOperand(MF, MMO);
1951 }
1952
storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI,Register VReg) const1953 void PPCInstrInfo::storeRegToStackSlot(
1954 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1955 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1956 const TargetRegisterInfo *TRI, Register VReg) const {
1957 // We need to avoid a situation in which the value from a VRRC register is
1958 // spilled using an Altivec instruction and reloaded into a VSRC register
1959 // using a VSX instruction. The issue with this is that the VSX
1960 // load/store instructions swap the doublewords in the vector and the Altivec
1961 // ones don't. The register classes on the spill/reload may be different if
1962 // the register is defined using an Altivec instruction and is then used by a
1963 // VSX instruction.
1964 RC = updatedRC(RC);
1965 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1966 }
1967
LoadRegFromStackSlot(MachineFunction & MF,const DebugLoc & DL,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1968 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1969 unsigned DestReg, int FrameIdx,
1970 const TargetRegisterClass *RC,
1971 SmallVectorImpl<MachineInstr *> &NewMIs)
1972 const {
1973 unsigned Opcode = getLoadOpcodeForSpill(RC);
1974 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1975 FrameIdx));
1976 }
1977
loadRegFromStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1978 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
1979 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
1980 int FrameIdx, const TargetRegisterClass *RC,
1981 const TargetRegisterInfo *TRI) const {
1982 MachineFunction &MF = *MBB.getParent();
1983 SmallVector<MachineInstr*, 4> NewMIs;
1984 DebugLoc DL;
1985 if (MI != MBB.end()) DL = MI->getDebugLoc();
1986
1987 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1988
1989 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1990 MBB.insert(MI, NewMIs[i]);
1991
1992 const MachineFrameInfo &MFI = MF.getFrameInfo();
1993 MachineMemOperand *MMO = MF.getMachineMemOperand(
1994 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1995 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1996 MFI.getObjectAlign(FrameIdx));
1997 NewMIs.back()->addMemOperand(MF, MMO);
1998 }
1999
loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI,Register VReg) const2000 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
2001 MachineBasicBlock::iterator MI,
2002 Register DestReg, int FrameIdx,
2003 const TargetRegisterClass *RC,
2004 const TargetRegisterInfo *TRI,
2005 Register VReg) const {
2006 // We need to avoid a situation in which the value from a VRRC register is
2007 // spilled using an Altivec instruction and reloaded into a VSRC register
2008 // using a VSX instruction. The issue with this is that the VSX
2009 // load/store instructions swap the doublewords in the vector and the Altivec
2010 // ones don't. The register classes on the spill/reload may be different if
2011 // the register is defined using an Altivec instruction and is then used by a
2012 // VSX instruction.
2013 RC = updatedRC(RC);
2014
2015 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2016 }
2017
2018 bool PPCInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const2019 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
2020 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2021 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2022 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2023 else
2024 // Leave the CR# the same, but invert the condition.
2025 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2026 return false;
2027 }
2028
2029 // For some instructions, it is legal to fold ZERO into the RA register field.
2030 // This function performs that fold by replacing the operand with PPC::ZERO,
2031 // it does not consider whether the load immediate zero is no longer in use.
onlyFoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg) const2032 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2033 Register Reg) const {
2034 // A zero immediate should always be loaded with a single li.
2035 unsigned DefOpc = DefMI.getOpcode();
2036 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2037 return false;
2038 if (!DefMI.getOperand(1).isImm())
2039 return false;
2040 if (DefMI.getOperand(1).getImm() != 0)
2041 return false;
2042
2043 // Note that we cannot here invert the arguments of an isel in order to fold
2044 // a ZERO into what is presented as the second argument. All we have here
2045 // is the condition bit, and that might come from a CR-logical bit operation.
2046
2047 const MCInstrDesc &UseMCID = UseMI.getDesc();
2048
2049 // Only fold into real machine instructions.
2050 if (UseMCID.isPseudo())
2051 return false;
2052
2053 // We need to find which of the User's operands is to be folded, that will be
2054 // the operand that matches the given register ID.
2055 unsigned UseIdx;
2056 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2057 if (UseMI.getOperand(UseIdx).isReg() &&
2058 UseMI.getOperand(UseIdx).getReg() == Reg)
2059 break;
2060
2061 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2062 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2063
2064 const MCOperandInfo *UseInfo = &UseMCID.operands()[UseIdx];
2065
2066 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2067 // register (which might also be specified as a pointer class kind).
2068 if (UseInfo->isLookupPtrRegClass()) {
2069 if (UseInfo->RegClass /* Kind */ != 1)
2070 return false;
2071 } else {
2072 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2073 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2074 return false;
2075 }
2076
2077 // Make sure this is not tied to an output register (or otherwise
2078 // constrained). This is true for ST?UX registers, for example, which
2079 // are tied to their output registers.
2080 if (UseInfo->Constraints != 0)
2081 return false;
2082
2083 MCRegister ZeroReg;
2084 if (UseInfo->isLookupPtrRegClass()) {
2085 bool isPPC64 = Subtarget.isPPC64();
2086 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2087 } else {
2088 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2089 PPC::ZERO8 : PPC::ZERO;
2090 }
2091
2092 LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2093 LLVM_DEBUG(UseMI.dump());
2094 UseMI.getOperand(UseIdx).setReg(ZeroReg);
2095 LLVM_DEBUG(dbgs() << "Into: ");
2096 LLVM_DEBUG(UseMI.dump());
2097 return true;
2098 }
2099
2100 // Folds zero into instructions which have a load immediate zero as an operand
2101 // but also recognize zero as immediate zero. If the definition of the load
2102 // has no more users it is deleted.
FoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg,MachineRegisterInfo * MRI) const2103 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2104 Register Reg, MachineRegisterInfo *MRI) const {
2105 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2106 if (MRI->use_nodbg_empty(Reg))
2107 DefMI.eraseFromParent();
2108 return Changed;
2109 }
2110
MBBDefinesCTR(MachineBasicBlock & MBB)2111 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
2112 for (MachineInstr &MI : MBB)
2113 if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2114 return true;
2115 return false;
2116 }
2117
2118 // We should make sure that, if we're going to predicate both sides of a
2119 // condition (a diamond), that both sides don't define the counter register. We
2120 // can predicate counter-decrement-based branches, but while that predicates
2121 // the branching, it does not predicate the counter decrement. If we tried to
2122 // merge the triangle into one predicated block, we'd decrement the counter
2123 // twice.
isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumT,unsigned ExtraT,MachineBasicBlock & FMBB,unsigned NumF,unsigned ExtraF,BranchProbability Probability) const2124 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
2125 unsigned NumT, unsigned ExtraT,
2126 MachineBasicBlock &FMBB,
2127 unsigned NumF, unsigned ExtraF,
2128 BranchProbability Probability) const {
2129 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2130 }
2131
2132
isPredicated(const MachineInstr & MI) const2133 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
2134 // The predicated branches are identified by their type, not really by the
2135 // explicit presence of a predicate. Furthermore, some of them can be
2136 // predicated more than once. Because if conversion won't try to predicate
2137 // any instruction which already claims to be predicated (by returning true
2138 // here), always return false. In doing so, we let isPredicable() be the
2139 // final word on whether not the instruction can be (further) predicated.
2140
2141 return false;
2142 }
2143
isSchedulingBoundary(const MachineInstr & MI,const MachineBasicBlock * MBB,const MachineFunction & MF) const2144 bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2145 const MachineBasicBlock *MBB,
2146 const MachineFunction &MF) const {
2147 switch (MI.getOpcode()) {
2148 default:
2149 break;
2150 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2151 // across them, since some FP operations may change content of FPSCR.
2152 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2153 case PPC::MFFS:
2154 case PPC::MTFSF:
2155 case PPC::FENCE:
2156 return true;
2157 }
2158 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
2159 }
2160
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const2161 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
2162 ArrayRef<MachineOperand> Pred) const {
2163 unsigned OpC = MI.getOpcode();
2164 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2165 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2166 bool isPPC64 = Subtarget.isPPC64();
2167 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2168 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2169 // Need add Def and Use for CTR implicit operand.
2170 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2171 .addReg(Pred[1].getReg(), RegState::Implicit)
2172 .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2173 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2174 MI.setDesc(get(PPC::BCLR));
2175 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2176 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2177 MI.setDesc(get(PPC::BCLRn));
2178 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2179 } else {
2180 MI.setDesc(get(PPC::BCCLR));
2181 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2182 .addImm(Pred[0].getImm())
2183 .add(Pred[1]);
2184 }
2185
2186 return true;
2187 } else if (OpC == PPC::B) {
2188 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2189 bool isPPC64 = Subtarget.isPPC64();
2190 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2191 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2192 // Need add Def and Use for CTR implicit operand.
2193 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2194 .addReg(Pred[1].getReg(), RegState::Implicit)
2195 .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2196 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2197 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2198 MI.removeOperand(0);
2199
2200 MI.setDesc(get(PPC::BC));
2201 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2202 .add(Pred[1])
2203 .addMBB(MBB);
2204 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2205 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2206 MI.removeOperand(0);
2207
2208 MI.setDesc(get(PPC::BCn));
2209 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2210 .add(Pred[1])
2211 .addMBB(MBB);
2212 } else {
2213 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2214 MI.removeOperand(0);
2215
2216 MI.setDesc(get(PPC::BCC));
2217 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2218 .addImm(Pred[0].getImm())
2219 .add(Pred[1])
2220 .addMBB(MBB);
2221 }
2222
2223 return true;
2224 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2225 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2226 OpC == PPC::BCTRL8_RM) {
2227 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2228 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2229
2230 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2231 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2232 bool isPPC64 = Subtarget.isPPC64();
2233
2234 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2235 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2236 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2237 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2238 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2239 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2240 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2241 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2242 } else {
2243 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2244 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2245 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2246 .addImm(Pred[0].getImm())
2247 .add(Pred[1]);
2248 }
2249
2250 // Need add Def and Use for LR implicit operand.
2251 if (setLR)
2252 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2253 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2254 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2255 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2256 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2257 .addReg(PPC::RM, RegState::ImplicitDefine);
2258
2259 return true;
2260 }
2261
2262 return false;
2263 }
2264
SubsumesPredicate(ArrayRef<MachineOperand> Pred1,ArrayRef<MachineOperand> Pred2) const2265 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
2266 ArrayRef<MachineOperand> Pred2) const {
2267 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2268 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2269
2270 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2271 return false;
2272 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2273 return false;
2274
2275 // P1 can only subsume P2 if they test the same condition register.
2276 if (Pred1[1].getReg() != Pred2[1].getReg())
2277 return false;
2278
2279 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2280 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2281
2282 if (P1 == P2)
2283 return true;
2284
2285 // Does P1 subsume P2, e.g. GE subsumes GT.
2286 if (P1 == PPC::PRED_LE &&
2287 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2288 return true;
2289 if (P1 == PPC::PRED_GE &&
2290 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2291 return true;
2292
2293 return false;
2294 }
2295
ClobbersPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred,bool SkipDead) const2296 bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
2297 std::vector<MachineOperand> &Pred,
2298 bool SkipDead) const {
2299 // Note: At the present time, the contents of Pred from this function is
2300 // unused by IfConversion. This implementation follows ARM by pushing the
2301 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2302 // predicate, instructions defining CTR or CTR8 are also included as
2303 // predicate-defining instructions.
2304
2305 const TargetRegisterClass *RCs[] =
2306 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2307 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2308
2309 bool Found = false;
2310 for (const MachineOperand &MO : MI.operands()) {
2311 for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2312 const TargetRegisterClass *RC = RCs[c];
2313 if (MO.isReg()) {
2314 if (MO.isDef() && RC->contains(MO.getReg())) {
2315 Pred.push_back(MO);
2316 Found = true;
2317 }
2318 } else if (MO.isRegMask()) {
2319 for (MCPhysReg R : *RC)
2320 if (MO.clobbersPhysReg(R)) {
2321 Pred.push_back(MO);
2322 Found = true;
2323 }
2324 }
2325 }
2326 }
2327
2328 return Found;
2329 }
2330
analyzeCompare(const MachineInstr & MI,Register & SrcReg,Register & SrcReg2,int64_t & Mask,int64_t & Value) const2331 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2332 Register &SrcReg2, int64_t &Mask,
2333 int64_t &Value) const {
2334 unsigned Opc = MI.getOpcode();
2335
2336 switch (Opc) {
2337 default: return false;
2338 case PPC::CMPWI:
2339 case PPC::CMPLWI:
2340 case PPC::CMPDI:
2341 case PPC::CMPLDI:
2342 SrcReg = MI.getOperand(1).getReg();
2343 SrcReg2 = 0;
2344 Value = MI.getOperand(2).getImm();
2345 Mask = 0xFFFF;
2346 return true;
2347 case PPC::CMPW:
2348 case PPC::CMPLW:
2349 case PPC::CMPD:
2350 case PPC::CMPLD:
2351 case PPC::FCMPUS:
2352 case PPC::FCMPUD:
2353 SrcReg = MI.getOperand(1).getReg();
2354 SrcReg2 = MI.getOperand(2).getReg();
2355 Value = 0;
2356 Mask = 0;
2357 return true;
2358 }
2359 }
2360
optimizeCompareInstr(MachineInstr & CmpInstr,Register SrcReg,Register SrcReg2,int64_t Mask,int64_t Value,const MachineRegisterInfo * MRI) const2361 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
2362 Register SrcReg2, int64_t Mask,
2363 int64_t Value,
2364 const MachineRegisterInfo *MRI) const {
2365 if (DisableCmpOpt)
2366 return false;
2367
2368 int OpC = CmpInstr.getOpcode();
2369 Register CRReg = CmpInstr.getOperand(0).getReg();
2370
2371 // FP record forms set CR1 based on the exception status bits, not a
2372 // comparison with zero.
2373 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2374 return false;
2375
2376 const TargetRegisterInfo *TRI = &getRegisterInfo();
2377 // The record forms set the condition register based on a signed comparison
2378 // with zero (so says the ISA manual). This is not as straightforward as it
2379 // seems, however, because this is always a 64-bit comparison on PPC64, even
2380 // for instructions that are 32-bit in nature (like slw for example).
2381 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2382 // for equality checks (as those don't depend on the sign). On PPC64,
2383 // we are restricted to equality for unsigned 64-bit comparisons and for
2384 // signed 32-bit comparisons the applicability is more restricted.
2385 bool isPPC64 = Subtarget.isPPC64();
2386 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2387 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2388 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2389
2390 // Look through copies unless that gets us to a physical register.
2391 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2392 if (ActualSrc.isVirtual())
2393 SrcReg = ActualSrc;
2394
2395 // Get the unique definition of SrcReg.
2396 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2397 if (!MI) return false;
2398
2399 bool equalityOnly = false;
2400 bool noSub = false;
2401 if (isPPC64) {
2402 if (is32BitSignedCompare) {
2403 // We can perform this optimization only if SrcReg is sign-extending.
2404 if (isSignExtended(SrcReg, MRI))
2405 noSub = true;
2406 else
2407 return false;
2408 } else if (is32BitUnsignedCompare) {
2409 // We can perform this optimization, equality only, if SrcReg is
2410 // zero-extending.
2411 if (isZeroExtended(SrcReg, MRI)) {
2412 noSub = true;
2413 equalityOnly = true;
2414 } else
2415 return false;
2416 } else
2417 equalityOnly = is64BitUnsignedCompare;
2418 } else
2419 equalityOnly = is32BitUnsignedCompare;
2420
2421 if (equalityOnly) {
2422 // We need to check the uses of the condition register in order to reject
2423 // non-equality comparisons.
2424 for (MachineRegisterInfo::use_instr_iterator
2425 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2426 I != IE; ++I) {
2427 MachineInstr *UseMI = &*I;
2428 if (UseMI->getOpcode() == PPC::BCC) {
2429 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2430 unsigned PredCond = PPC::getPredicateCondition(Pred);
2431 // We ignore hint bits when checking for non-equality comparisons.
2432 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2433 return false;
2434 } else if (UseMI->getOpcode() == PPC::ISEL ||
2435 UseMI->getOpcode() == PPC::ISEL8) {
2436 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2437 if (SubIdx != PPC::sub_eq)
2438 return false;
2439 } else
2440 return false;
2441 }
2442 }
2443
2444 MachineBasicBlock::iterator I = CmpInstr;
2445
2446 // Scan forward to find the first use of the compare.
2447 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2448 ++I) {
2449 bool FoundUse = false;
2450 for (MachineRegisterInfo::use_instr_iterator
2451 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2452 J != JE; ++J)
2453 if (&*J == &*I) {
2454 FoundUse = true;
2455 break;
2456 }
2457
2458 if (FoundUse)
2459 break;
2460 }
2461
2462 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
2463 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
2464
2465 // There are two possible candidates which can be changed to set CR[01].
2466 // One is MI, the other is a SUB instruction.
2467 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2468 MachineInstr *Sub = nullptr;
2469 if (SrcReg2 != 0)
2470 // MI is not a candidate for CMPrr.
2471 MI = nullptr;
2472 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2473 // same BB as the comparison. This is to allow the check below to avoid calls
2474 // (and other explicit clobbers); instead we should really check for these
2475 // more explicitly (in at least a few predecessors).
2476 else if (MI->getParent() != CmpInstr.getParent())
2477 return false;
2478 else if (Value != 0) {
2479 // The record-form instructions set CR bit based on signed comparison
2480 // against 0. We try to convert a compare against 1 or -1 into a compare
2481 // against 0 to exploit record-form instructions. For example, we change
2482 // the condition "greater than -1" into "greater than or equal to 0"
2483 // and "less than 1" into "less than or equal to 0".
2484
2485 // Since we optimize comparison based on a specific branch condition,
2486 // we don't optimize if condition code is used by more than once.
2487 if (equalityOnly || !MRI->hasOneUse(CRReg))
2488 return false;
2489
2490 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2491 if (UseMI->getOpcode() != PPC::BCC)
2492 return false;
2493
2494 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2495 unsigned PredCond = PPC::getPredicateCondition(Pred);
2496 unsigned PredHint = PPC::getPredicateHint(Pred);
2497 int16_t Immed = (int16_t)Value;
2498
2499 // When modifying the condition in the predicate, we propagate hint bits
2500 // from the original predicate to the new one.
2501 if (Immed == -1 && PredCond == PPC::PRED_GT)
2502 // We convert "greater than -1" into "greater than or equal to 0",
2503 // since we are assuming signed comparison by !equalityOnly
2504 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2505 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2506 // We convert "less than or equal to -1" into "less than 0".
2507 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2508 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2509 // We convert "less than 1" into "less than or equal to 0".
2510 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2511 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2512 // We convert "greater than or equal to 1" into "greater than 0".
2513 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2514 else
2515 return false;
2516
2517 // Convert the comparison and its user to a compare against zero with the
2518 // appropriate predicate on the branch. Zero comparison might provide
2519 // optimization opportunities post-RA (see optimization in
2520 // PPCPreEmitPeephole.cpp).
2521 UseMI->getOperand(0).setImm(Pred);
2522 CmpInstr.getOperand(2).setImm(0);
2523 }
2524
2525 // Search for Sub.
2526 --I;
2527
2528 // Get ready to iterate backward from CmpInstr.
2529 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2530
2531 for (; I != E && !noSub; --I) {
2532 const MachineInstr &Instr = *I;
2533 unsigned IOpC = Instr.getOpcode();
2534
2535 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2536 Instr.readsRegister(PPC::CR0, TRI)))
2537 // This instruction modifies or uses the record condition register after
2538 // the one we want to change. While we could do this transformation, it
2539 // would likely not be profitable. This transformation removes one
2540 // instruction, and so even forcing RA to generate one move probably
2541 // makes it unprofitable.
2542 return false;
2543
2544 // Check whether CmpInstr can be made redundant by the current instruction.
2545 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2546 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2547 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2548 ((Instr.getOperand(1).getReg() == SrcReg &&
2549 Instr.getOperand(2).getReg() == SrcReg2) ||
2550 (Instr.getOperand(1).getReg() == SrcReg2 &&
2551 Instr.getOperand(2).getReg() == SrcReg))) {
2552 Sub = &*I;
2553 break;
2554 }
2555
2556 if (I == B)
2557 // The 'and' is below the comparison instruction.
2558 return false;
2559 }
2560
2561 // Return false if no candidates exist.
2562 if (!MI && !Sub)
2563 return false;
2564
2565 // The single candidate is called MI.
2566 if (!MI) MI = Sub;
2567
2568 int NewOpC = -1;
2569 int MIOpC = MI->getOpcode();
2570 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2571 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2572 NewOpC = MIOpC;
2573 else {
2574 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2575 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2576 NewOpC = MIOpC;
2577 }
2578
2579 // FIXME: On the non-embedded POWER architectures, only some of the record
2580 // forms are fast, and we should use only the fast ones.
2581
2582 // The defining instruction has a record form (or is already a record
2583 // form). It is possible, however, that we'll need to reverse the condition
2584 // code of the users.
2585 if (NewOpC == -1)
2586 return false;
2587
2588 // This transformation should not be performed if `nsw` is missing and is not
2589 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2590 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2591 // CRReg can reflect if compared values are equal, this optz is still valid.
2592 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2593 Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2594 return false;
2595
2596 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2597 // needs to be updated to be based on SUB. Push the condition code
2598 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2599 // condition code of these operands will be modified.
2600 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2601 // comparison against 0, which may modify predicate.
2602 bool ShouldSwap = false;
2603 if (Sub && Value == 0) {
2604 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2605 Sub->getOperand(2).getReg() == SrcReg;
2606
2607 // The operands to subf are the opposite of sub, so only in the fixed-point
2608 // case, invert the order.
2609 ShouldSwap = !ShouldSwap;
2610 }
2611
2612 if (ShouldSwap)
2613 for (MachineRegisterInfo::use_instr_iterator
2614 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2615 I != IE; ++I) {
2616 MachineInstr *UseMI = &*I;
2617 if (UseMI->getOpcode() == PPC::BCC) {
2618 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2619 unsigned PredCond = PPC::getPredicateCondition(Pred);
2620 assert((!equalityOnly ||
2621 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2622 "Invalid predicate for equality-only optimization");
2623 (void)PredCond; // To suppress warning in release build.
2624 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2625 PPC::getSwappedPredicate(Pred)));
2626 } else if (UseMI->getOpcode() == PPC::ISEL ||
2627 UseMI->getOpcode() == PPC::ISEL8) {
2628 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2629 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2630 "Invalid CR bit for equality-only optimization");
2631
2632 if (NewSubReg == PPC::sub_lt)
2633 NewSubReg = PPC::sub_gt;
2634 else if (NewSubReg == PPC::sub_gt)
2635 NewSubReg = PPC::sub_lt;
2636
2637 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2638 NewSubReg));
2639 } else // We need to abort on a user we don't understand.
2640 return false;
2641 }
2642 assert(!(Value != 0 && ShouldSwap) &&
2643 "Non-zero immediate support and ShouldSwap"
2644 "may conflict in updating predicate");
2645
2646 // Create a new virtual register to hold the value of the CR set by the
2647 // record-form instruction. If the instruction was not previously in
2648 // record form, then set the kill flag on the CR.
2649 CmpInstr.eraseFromParent();
2650
2651 MachineBasicBlock::iterator MII = MI;
2652 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2653 get(TargetOpcode::COPY), CRReg)
2654 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2655
2656 // Even if CR0 register were dead before, it is alive now since the
2657 // instruction we just built uses it.
2658 MI->clearRegisterDeads(PPC::CR0);
2659
2660 if (MIOpC != NewOpC) {
2661 // We need to be careful here: we're replacing one instruction with
2662 // another, and we need to make sure that we get all of the right
2663 // implicit uses and defs. On the other hand, the caller may be holding
2664 // an iterator to this instruction, and so we can't delete it (this is
2665 // specifically the case if this is the instruction directly after the
2666 // compare).
2667
2668 // Rotates are expensive instructions. If we're emitting a record-form
2669 // rotate that can just be an andi/andis, we should just emit that.
2670 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2671 Register GPRRes = MI->getOperand(0).getReg();
2672 int64_t SH = MI->getOperand(2).getImm();
2673 int64_t MB = MI->getOperand(3).getImm();
2674 int64_t ME = MI->getOperand(4).getImm();
2675 // We can only do this if both the start and end of the mask are in the
2676 // same halfword.
2677 bool MBInLoHWord = MB >= 16;
2678 bool MEInLoHWord = ME >= 16;
2679 uint64_t Mask = ~0LLU;
2680
2681 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2682 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2683 // The mask value needs to shift right 16 if we're emitting andis.
2684 Mask >>= MBInLoHWord ? 0 : 16;
2685 NewOpC = MIOpC == PPC::RLWINM
2686 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2687 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2688 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2689 (ME - MB + 1 == SH) && (MB >= 16)) {
2690 // If we are rotating by the exact number of bits as are in the mask
2691 // and the mask is in the least significant bits of the register,
2692 // that's just an andis. (as long as the GPR result has no uses).
2693 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2694 Mask >>= 16;
2695 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2696 }
2697 // If we've set the mask, we can transform.
2698 if (Mask != ~0LLU) {
2699 MI->removeOperand(4);
2700 MI->removeOperand(3);
2701 MI->getOperand(2).setImm(Mask);
2702 NumRcRotatesConvertedToRcAnd++;
2703 }
2704 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2705 int64_t MB = MI->getOperand(3).getImm();
2706 if (MB >= 48) {
2707 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2708 NewOpC = PPC::ANDI8_rec;
2709 MI->removeOperand(3);
2710 MI->getOperand(2).setImm(Mask);
2711 NumRcRotatesConvertedToRcAnd++;
2712 }
2713 }
2714
2715 const MCInstrDesc &NewDesc = get(NewOpC);
2716 MI->setDesc(NewDesc);
2717
2718 for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2719 if (!MI->definesRegister(ImpDef)) {
2720 MI->addOperand(*MI->getParent()->getParent(),
2721 MachineOperand::CreateReg(ImpDef, true, true));
2722 }
2723 }
2724 for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2725 if (!MI->readsRegister(ImpUse)) {
2726 MI->addOperand(*MI->getParent()->getParent(),
2727 MachineOperand::CreateReg(ImpUse, false, true));
2728 }
2729 }
2730 }
2731 assert(MI->definesRegister(PPC::CR0) &&
2732 "Record-form instruction does not define cr0?");
2733
2734 // Modify the condition code of operands in OperandsToUpdate.
2735 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2736 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2737 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2738 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2739
2740 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2741 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2742
2743 return true;
2744 }
2745
optimizeCmpPostRA(MachineInstr & CmpMI) const2746 bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
2747 MachineRegisterInfo *MRI = &CmpMI.getParent()->getParent()->getRegInfo();
2748 if (MRI->isSSA())
2749 return false;
2750
2751 Register SrcReg, SrcReg2;
2752 int64_t CmpMask, CmpValue;
2753 if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
2754 return false;
2755
2756 // Try to optimize the comparison against 0.
2757 if (CmpValue || !CmpMask || SrcReg2)
2758 return false;
2759
2760 // The record forms set the condition register based on a signed comparison
2761 // with zero (see comments in optimizeCompareInstr). Since we can't do the
2762 // equality checks in post-RA, we are more restricted on a unsigned
2763 // comparison.
2764 unsigned Opc = CmpMI.getOpcode();
2765 if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2766 return false;
2767
2768 // The record forms are always based on a 64-bit comparison on PPC64
2769 // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2770 // comparison. Since we can't do the equality checks in post-RA, we bail out
2771 // the case.
2772 if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2773 return false;
2774
2775 // CmpMI can't be deleted if it has implicit def.
2776 if (CmpMI.hasImplicitDef())
2777 return false;
2778
2779 bool SrcRegHasOtherUse = false;
2780 MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
2781 if (!SrcMI || !SrcMI->definesRegister(SrcReg))
2782 return false;
2783
2784 MachineOperand RegMO = CmpMI.getOperand(0);
2785 Register CRReg = RegMO.getReg();
2786 if (CRReg != PPC::CR0)
2787 return false;
2788
2789 // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2790 bool SeenUseOfCRReg = false;
2791 bool IsCRRegKilled = false;
2792 if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
2793 SeenUseOfCRReg) ||
2794 SrcMI->definesRegister(CRReg) || SeenUseOfCRReg)
2795 return false;
2796
2797 int SrcMIOpc = SrcMI->getOpcode();
2798 int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
2799 if (NewOpC == -1)
2800 return false;
2801
2802 LLVM_DEBUG(dbgs() << "Replace Instr: ");
2803 LLVM_DEBUG(SrcMI->dump());
2804
2805 const MCInstrDesc &NewDesc = get(NewOpC);
2806 SrcMI->setDesc(NewDesc);
2807 MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2808 .addReg(CRReg, RegState::ImplicitDefine);
2809 SrcMI->clearRegisterDeads(CRReg);
2810
2811 assert(SrcMI->definesRegister(PPC::CR0) &&
2812 "Record-form instruction does not define cr0?");
2813
2814 LLVM_DEBUG(dbgs() << "with: ");
2815 LLVM_DEBUG(SrcMI->dump());
2816 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2817 LLVM_DEBUG(CmpMI.dump());
2818 return true;
2819 }
2820
getMemOperandsWithOffsetWidth(const MachineInstr & LdSt,SmallVectorImpl<const MachineOperand * > & BaseOps,int64_t & Offset,bool & OffsetIsScalable,unsigned & Width,const TargetRegisterInfo * TRI) const2821 bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
2822 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2823 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2824 const TargetRegisterInfo *TRI) const {
2825 const MachineOperand *BaseOp;
2826 OffsetIsScalable = false;
2827 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2828 return false;
2829 BaseOps.push_back(BaseOp);
2830 return true;
2831 }
2832
isLdStSafeToCluster(const MachineInstr & LdSt,const TargetRegisterInfo * TRI)2833 static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2834 const TargetRegisterInfo *TRI) {
2835 // If this is a volatile load/store, don't mess with it.
2836 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2837 return false;
2838
2839 if (LdSt.getOperand(2).isFI())
2840 return true;
2841
2842 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2843 // Can't cluster if the instruction modifies the base register
2844 // or it is update form. e.g. ld r2,3(r2)
2845 if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2846 return false;
2847
2848 return true;
2849 }
2850
2851 // Only cluster instruction pair that have the same opcode, and they are
2852 // clusterable according to PowerPC specification.
isClusterableLdStOpcPair(unsigned FirstOpc,unsigned SecondOpc,const PPCSubtarget & Subtarget)2853 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2854 const PPCSubtarget &Subtarget) {
2855 switch (FirstOpc) {
2856 default:
2857 return false;
2858 case PPC::STD:
2859 case PPC::STFD:
2860 case PPC::STXSD:
2861 case PPC::DFSTOREf64:
2862 return FirstOpc == SecondOpc;
2863 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2864 // 32bit and 64bit instruction selection. They are clusterable pair though
2865 // they are different opcode.
2866 case PPC::STW:
2867 case PPC::STW8:
2868 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2869 }
2870 }
2871
shouldClusterMemOps(ArrayRef<const MachineOperand * > BaseOps1,int64_t OpOffset1,bool OffsetIsScalable1,ArrayRef<const MachineOperand * > BaseOps2,int64_t OpOffset2,bool OffsetIsScalable2,unsigned ClusterSize,unsigned NumBytes) const2872 bool PPCInstrInfo::shouldClusterMemOps(
2873 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
2874 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2875 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
2876 unsigned NumBytes) const {
2877
2878 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2879 const MachineOperand &BaseOp1 = *BaseOps1.front();
2880 const MachineOperand &BaseOp2 = *BaseOps2.front();
2881 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2882 "Only base registers and frame indices are supported.");
2883
2884 // ClusterSize means the number of memory operations that will have been
2885 // clustered if this hook returns true.
2886 // Don't cluster memory op if there are already two ops clustered at least.
2887 if (ClusterSize > 2)
2888 return false;
2889
2890 // Cluster the load/store only when they have the same base
2891 // register or FI.
2892 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2893 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2894 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2895 return false;
2896
2897 // Check if the load/store are clusterable according to the PowerPC
2898 // specification.
2899 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2900 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2901 unsigned FirstOpc = FirstLdSt.getOpcode();
2902 unsigned SecondOpc = SecondLdSt.getOpcode();
2903 const TargetRegisterInfo *TRI = &getRegisterInfo();
2904 // Cluster the load/store only when they have the same opcode, and they are
2905 // clusterable opcode according to PowerPC specification.
2906 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2907 return false;
2908
2909 // Can't cluster load/store that have ordered or volatile memory reference.
2910 if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2911 !isLdStSafeToCluster(SecondLdSt, TRI))
2912 return false;
2913
2914 int64_t Offset1 = 0, Offset2 = 0;
2915 unsigned Width1 = 0, Width2 = 0;
2916 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2917 if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2918 !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2919 Width1 != Width2)
2920 return false;
2921
2922 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2923 "getMemOperandWithOffsetWidth return incorrect base op");
2924 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2925 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2926 return Offset1 + Width1 == Offset2;
2927 }
2928
2929 /// GetInstSize - Return the number of bytes of code the specified
2930 /// instruction may be. This returns the maximum number of bytes.
2931 ///
getInstSizeInBytes(const MachineInstr & MI) const2932 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2933 unsigned Opcode = MI.getOpcode();
2934
2935 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2936 const MachineFunction *MF = MI.getParent()->getParent();
2937 const char *AsmStr = MI.getOperand(0).getSymbolName();
2938 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2939 } else if (Opcode == TargetOpcode::STACKMAP) {
2940 StackMapOpers Opers(&MI);
2941 return Opers.getNumPatchBytes();
2942 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2943 PatchPointOpers Opers(&MI);
2944 return Opers.getNumPatchBytes();
2945 } else {
2946 return get(Opcode).getSize();
2947 }
2948 }
2949
2950 std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const2951 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2952 // PPC always uses a direct mask.
2953 return std::make_pair(TF, 0u);
2954 }
2955
2956 ArrayRef<std::pair<unsigned, const char *>>
getSerializableDirectMachineOperandTargetFlags() const2957 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2958 using namespace PPCII;
2959 static const std::pair<unsigned, const char *> TargetFlags[] = {
2960 {MO_PLT, "ppc-plt"},
2961 {MO_PIC_FLAG, "ppc-pic"},
2962 {MO_PCREL_FLAG, "ppc-pcrel"},
2963 {MO_GOT_FLAG, "ppc-got"},
2964 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
2965 {MO_TLSGD_FLAG, "ppc-tlsgd"},
2966 {MO_TPREL_FLAG, "ppc-tprel"},
2967 {MO_TLSLD_FLAG, "ppc-tlsld"},
2968 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
2969 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
2970 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
2971 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"},
2972 {MO_LO, "ppc-lo"},
2973 {MO_HA, "ppc-ha"},
2974 {MO_TPREL_LO, "ppc-tprel-lo"},
2975 {MO_TPREL_HA, "ppc-tprel-ha"},
2976 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2977 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2978 {MO_TOC_LO, "ppc-toc-lo"},
2979 {MO_TLS, "ppc-tls"},
2980 {MO_PIC_HA_FLAG, "ppc-ha-pic"},
2981 {MO_PIC_LO_FLAG, "ppc-lo-pic"},
2982 {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"},
2983 {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"},
2984 {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"},
2985 };
2986 return ArrayRef(TargetFlags);
2987 }
2988
2989 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2990 // The VSX versions have the advantage of a full 64-register target whereas
2991 // the FP ones have the advantage of lower latency and higher throughput. So
2992 // what we are after is using the faster instructions in low register pressure
2993 // situations and using the larger register file in high register pressure
2994 // situations.
expandVSXMemPseudo(MachineInstr & MI) const2995 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2996 unsigned UpperOpcode, LowerOpcode;
2997 switch (MI.getOpcode()) {
2998 case PPC::DFLOADf32:
2999 UpperOpcode = PPC::LXSSP;
3000 LowerOpcode = PPC::LFS;
3001 break;
3002 case PPC::DFLOADf64:
3003 UpperOpcode = PPC::LXSD;
3004 LowerOpcode = PPC::LFD;
3005 break;
3006 case PPC::DFSTOREf32:
3007 UpperOpcode = PPC::STXSSP;
3008 LowerOpcode = PPC::STFS;
3009 break;
3010 case PPC::DFSTOREf64:
3011 UpperOpcode = PPC::STXSD;
3012 LowerOpcode = PPC::STFD;
3013 break;
3014 case PPC::XFLOADf32:
3015 UpperOpcode = PPC::LXSSPX;
3016 LowerOpcode = PPC::LFSX;
3017 break;
3018 case PPC::XFLOADf64:
3019 UpperOpcode = PPC::LXSDX;
3020 LowerOpcode = PPC::LFDX;
3021 break;
3022 case PPC::XFSTOREf32:
3023 UpperOpcode = PPC::STXSSPX;
3024 LowerOpcode = PPC::STFSX;
3025 break;
3026 case PPC::XFSTOREf64:
3027 UpperOpcode = PPC::STXSDX;
3028 LowerOpcode = PPC::STFDX;
3029 break;
3030 case PPC::LIWAX:
3031 UpperOpcode = PPC::LXSIWAX;
3032 LowerOpcode = PPC::LFIWAX;
3033 break;
3034 case PPC::LIWZX:
3035 UpperOpcode = PPC::LXSIWZX;
3036 LowerOpcode = PPC::LFIWZX;
3037 break;
3038 case PPC::STIWX:
3039 UpperOpcode = PPC::STXSIWX;
3040 LowerOpcode = PPC::STFIWX;
3041 break;
3042 default:
3043 llvm_unreachable("Unknown Operation!");
3044 }
3045
3046 Register TargetReg = MI.getOperand(0).getReg();
3047 unsigned Opcode;
3048 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3049 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3050 Opcode = LowerOpcode;
3051 else
3052 Opcode = UpperOpcode;
3053 MI.setDesc(get(Opcode));
3054 return true;
3055 }
3056
isAnImmediateOperand(const MachineOperand & MO)3057 static bool isAnImmediateOperand(const MachineOperand &MO) {
3058 return MO.isCPI() || MO.isGlobal() || MO.isImm();
3059 }
3060
expandPostRAPseudo(MachineInstr & MI) const3061 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
3062 auto &MBB = *MI.getParent();
3063 auto DL = MI.getDebugLoc();
3064
3065 switch (MI.getOpcode()) {
3066 case PPC::BUILD_UACC: {
3067 MCRegister ACC = MI.getOperand(0).getReg();
3068 MCRegister UACC = MI.getOperand(1).getReg();
3069 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3070 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3071 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3072 // FIXME: This can easily be improved to look up to the top of the MBB
3073 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3074 // we can just re-target any such XXLOR's to DstVSR + offset.
3075 for (int VecNo = 0; VecNo < 4; VecNo++)
3076 BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3077 .addReg(SrcVSR + VecNo)
3078 .addReg(SrcVSR + VecNo);
3079 }
3080 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3081 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3082 // with a NOP.
3083 [[fallthrough]];
3084 }
3085 case PPC::KILL_PAIR: {
3086 MI.setDesc(get(PPC::UNENCODED_NOP));
3087 MI.removeOperand(1);
3088 MI.removeOperand(0);
3089 return true;
3090 }
3091 case TargetOpcode::LOAD_STACK_GUARD: {
3092 assert(Subtarget.isTargetLinux() &&
3093 "Only Linux target is expected to contain LOAD_STACK_GUARD");
3094 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3095 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3096 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3097 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3098 .addImm(Offset)
3099 .addReg(Reg);
3100 return true;
3101 }
3102 case PPC::DFLOADf32:
3103 case PPC::DFLOADf64:
3104 case PPC::DFSTOREf32:
3105 case PPC::DFSTOREf64: {
3106 assert(Subtarget.hasP9Vector() &&
3107 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3108 assert(MI.getOperand(2).isReg() &&
3109 isAnImmediateOperand(MI.getOperand(1)) &&
3110 "D-form op must have register and immediate operands");
3111 return expandVSXMemPseudo(MI);
3112 }
3113 case PPC::XFLOADf32:
3114 case PPC::XFSTOREf32:
3115 case PPC::LIWAX:
3116 case PPC::LIWZX:
3117 case PPC::STIWX: {
3118 assert(Subtarget.hasP8Vector() &&
3119 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3120 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3121 "X-form op must have register and register operands");
3122 return expandVSXMemPseudo(MI);
3123 }
3124 case PPC::XFLOADf64:
3125 case PPC::XFSTOREf64: {
3126 assert(Subtarget.hasVSX() &&
3127 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3128 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3129 "X-form op must have register and register operands");
3130 return expandVSXMemPseudo(MI);
3131 }
3132 case PPC::SPILLTOVSR_LD: {
3133 Register TargetReg = MI.getOperand(0).getReg();
3134 if (PPC::VSFRCRegClass.contains(TargetReg)) {
3135 MI.setDesc(get(PPC::DFLOADf64));
3136 return expandPostRAPseudo(MI);
3137 }
3138 else
3139 MI.setDesc(get(PPC::LD));
3140 return true;
3141 }
3142 case PPC::SPILLTOVSR_ST: {
3143 Register SrcReg = MI.getOperand(0).getReg();
3144 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3145 NumStoreSPILLVSRRCAsVec++;
3146 MI.setDesc(get(PPC::DFSTOREf64));
3147 return expandPostRAPseudo(MI);
3148 } else {
3149 NumStoreSPILLVSRRCAsGpr++;
3150 MI.setDesc(get(PPC::STD));
3151 }
3152 return true;
3153 }
3154 case PPC::SPILLTOVSR_LDX: {
3155 Register TargetReg = MI.getOperand(0).getReg();
3156 if (PPC::VSFRCRegClass.contains(TargetReg))
3157 MI.setDesc(get(PPC::LXSDX));
3158 else
3159 MI.setDesc(get(PPC::LDX));
3160 return true;
3161 }
3162 case PPC::SPILLTOVSR_STX: {
3163 Register SrcReg = MI.getOperand(0).getReg();
3164 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3165 NumStoreSPILLVSRRCAsVec++;
3166 MI.setDesc(get(PPC::STXSDX));
3167 } else {
3168 NumStoreSPILLVSRRCAsGpr++;
3169 MI.setDesc(get(PPC::STDX));
3170 }
3171 return true;
3172 }
3173
3174 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3175 case PPC::CFENCE:
3176 case PPC::CFENCE8: {
3177 auto Val = MI.getOperand(0).getReg();
3178 unsigned CmpOp = Subtarget.isPPC64() ? PPC::CMPD : PPC::CMPW;
3179 BuildMI(MBB, MI, DL, get(CmpOp), PPC::CR7).addReg(Val).addReg(Val);
3180 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3181 .addImm(PPC::PRED_NE_MINUS)
3182 .addReg(PPC::CR7)
3183 .addImm(1);
3184 MI.setDesc(get(PPC::ISYNC));
3185 MI.removeOperand(0);
3186 return true;
3187 }
3188 }
3189 return false;
3190 }
3191
3192 // Essentially a compile-time implementation of a compare->isel sequence.
3193 // It takes two constants to compare, along with the true/false registers
3194 // and the comparison type (as a subreg to a CR field) and returns one
3195 // of the true/false registers, depending on the comparison results.
selectReg(int64_t Imm1,int64_t Imm2,unsigned CompareOpc,unsigned TrueReg,unsigned FalseReg,unsigned CRSubReg)3196 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3197 unsigned TrueReg, unsigned FalseReg,
3198 unsigned CRSubReg) {
3199 // Signed comparisons. The immediates are assumed to be sign-extended.
3200 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3201 switch (CRSubReg) {
3202 default: llvm_unreachable("Unknown integer comparison type.");
3203 case PPC::sub_lt:
3204 return Imm1 < Imm2 ? TrueReg : FalseReg;
3205 case PPC::sub_gt:
3206 return Imm1 > Imm2 ? TrueReg : FalseReg;
3207 case PPC::sub_eq:
3208 return Imm1 == Imm2 ? TrueReg : FalseReg;
3209 }
3210 }
3211 // Unsigned comparisons.
3212 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3213 switch (CRSubReg) {
3214 default: llvm_unreachable("Unknown integer comparison type.");
3215 case PPC::sub_lt:
3216 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3217 case PPC::sub_gt:
3218 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3219 case PPC::sub_eq:
3220 return Imm1 == Imm2 ? TrueReg : FalseReg;
3221 }
3222 }
3223 return PPC::NoRegister;
3224 }
3225
replaceInstrOperandWithImm(MachineInstr & MI,unsigned OpNo,int64_t Imm) const3226 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
3227 unsigned OpNo,
3228 int64_t Imm) const {
3229 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3230 // Replace the REG with the Immediate.
3231 Register InUseReg = MI.getOperand(OpNo).getReg();
3232 MI.getOperand(OpNo).ChangeToImmediate(Imm);
3233
3234 // We need to make sure that the MI didn't have any implicit use
3235 // of this REG any more. We don't call MI.implicit_operands().empty() to
3236 // return early, since MI's MCID might be changed in calling context, as a
3237 // result its number of explicit operands may be changed, thus the begin of
3238 // implicit operand is changed.
3239 const TargetRegisterInfo *TRI = &getRegisterInfo();
3240 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3241 if (UseOpIdx >= 0) {
3242 MachineOperand &MO = MI.getOperand(UseOpIdx);
3243 if (MO.isImplicit())
3244 // The operands must always be in the following order:
3245 // - explicit reg defs,
3246 // - other explicit operands (reg uses, immediates, etc.),
3247 // - implicit reg defs
3248 // - implicit reg uses
3249 // Therefore, removing the implicit operand won't change the explicit
3250 // operands layout.
3251 MI.removeOperand(UseOpIdx);
3252 }
3253 }
3254
3255 // Replace an instruction with one that materializes a constant (and sets
3256 // CR0 if the original instruction was a record-form instruction).
replaceInstrWithLI(MachineInstr & MI,const LoadImmediateInfo & LII) const3257 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
3258 const LoadImmediateInfo &LII) const {
3259 // Remove existing operands.
3260 int OperandToKeep = LII.SetCR ? 1 : 0;
3261 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3262 MI.removeOperand(i);
3263
3264 // Replace the instruction.
3265 if (LII.SetCR) {
3266 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3267 // Set the immediate.
3268 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3269 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3270 return;
3271 }
3272 else
3273 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3274
3275 // Set the immediate.
3276 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3277 .addImm(LII.Imm);
3278 }
3279
getDefMIPostRA(unsigned Reg,MachineInstr & MI,bool & SeenIntermediateUse) const3280 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
3281 bool &SeenIntermediateUse) const {
3282 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3283 "Should be called after register allocation.");
3284 const TargetRegisterInfo *TRI = &getRegisterInfo();
3285 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3286 It++;
3287 SeenIntermediateUse = false;
3288 for (; It != E; ++It) {
3289 if (It->modifiesRegister(Reg, TRI))
3290 return &*It;
3291 if (It->readsRegister(Reg, TRI))
3292 SeenIntermediateUse = true;
3293 }
3294 return nullptr;
3295 }
3296
materializeImmPostRA(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,Register Reg,int64_t Imm) const3297 void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB,
3298 MachineBasicBlock::iterator MBBI,
3299 const DebugLoc &DL, Register Reg,
3300 int64_t Imm) const {
3301 assert(!MBB.getParent()->getRegInfo().isSSA() &&
3302 "Register should be in non-SSA form after RA");
3303 bool isPPC64 = Subtarget.isPPC64();
3304 // FIXME: Materialization here is not optimal.
3305 // For some special bit patterns we can use less instructions.
3306 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3307 if (isInt<16>(Imm)) {
3308 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3309 } else if (isInt<32>(Imm)) {
3310 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3311 .addImm(Imm >> 16);
3312 if (Imm & 0xFFFF)
3313 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3314 .addReg(Reg, RegState::Kill)
3315 .addImm(Imm & 0xFFFF);
3316 } else {
3317 assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3318 "only supported in PPC64");
3319 BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3320 if ((Imm >> 32) & 0xFFFF)
3321 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3322 .addReg(Reg, RegState::Kill)
3323 .addImm((Imm >> 32) & 0xFFFF);
3324 BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3325 .addReg(Reg, RegState::Kill)
3326 .addImm(32)
3327 .addImm(31);
3328 BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3329 .addReg(Reg, RegState::Kill)
3330 .addImm((Imm >> 16) & 0xFFFF);
3331 if (Imm & 0xFFFF)
3332 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3333 .addReg(Reg, RegState::Kill)
3334 .addImm(Imm & 0xFFFF);
3335 }
3336 }
3337
getForwardingDefMI(MachineInstr & MI,unsigned & OpNoForForwarding,bool & SeenIntermediateUse) const3338 MachineInstr *PPCInstrInfo::getForwardingDefMI(
3339 MachineInstr &MI,
3340 unsigned &OpNoForForwarding,
3341 bool &SeenIntermediateUse) const {
3342 OpNoForForwarding = ~0U;
3343 MachineInstr *DefMI = nullptr;
3344 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3345 const TargetRegisterInfo *TRI = &getRegisterInfo();
3346 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3347 // within the basic block to see if the register is defined using an
3348 // LI/LI8/ADDI/ADDI8.
3349 if (MRI->isSSA()) {
3350 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3351 if (!MI.getOperand(i).isReg())
3352 continue;
3353 Register Reg = MI.getOperand(i).getReg();
3354 if (!Reg.isVirtual())
3355 continue;
3356 Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3357 if (TrueReg.isVirtual()) {
3358 MachineInstr *DefMIForTrueReg = MRI->getVRegDef(TrueReg);
3359 if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3360 DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3361 DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3362 DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3363 OpNoForForwarding = i;
3364 DefMI = DefMIForTrueReg;
3365 // The ADDI and LI operand maybe exist in one instruction at same
3366 // time. we prefer to fold LI operand as LI only has one Imm operand
3367 // and is more possible to be converted. So if current DefMI is
3368 // ADDI/ADDI8, we continue to find possible LI/LI8.
3369 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3370 break;
3371 }
3372 }
3373 }
3374 } else {
3375 // Looking back through the definition for each operand could be expensive,
3376 // so exit early if this isn't an instruction that either has an immediate
3377 // form or is already an immediate form that we can handle.
3378 ImmInstrInfo III;
3379 unsigned Opc = MI.getOpcode();
3380 bool ConvertibleImmForm =
3381 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3382 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3383 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3384 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3385 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3386 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3387 Opc == PPC::RLWINM8_rec;
3388 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3389 ? PPC::isVFRegister(MI.getOperand(0).getReg())
3390 : false;
3391 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3392 return nullptr;
3393
3394 // Don't convert or %X, %Y, %Y since that's just a register move.
3395 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3396 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3397 return nullptr;
3398 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3399 MachineOperand &MO = MI.getOperand(i);
3400 SeenIntermediateUse = false;
3401 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3402 Register Reg = MI.getOperand(i).getReg();
3403 // If we see another use of this reg between the def and the MI,
3404 // we want to flag it so the def isn't deleted.
3405 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3406 if (DefMI) {
3407 // Is this register defined by some form of add-immediate (including
3408 // load-immediate) within this basic block?
3409 switch (DefMI->getOpcode()) {
3410 default:
3411 break;
3412 case PPC::LI:
3413 case PPC::LI8:
3414 case PPC::ADDItocL:
3415 case PPC::ADDI:
3416 case PPC::ADDI8:
3417 OpNoForForwarding = i;
3418 return DefMI;
3419 }
3420 }
3421 }
3422 }
3423 }
3424 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3425 }
3426
getSpillTarget() const3427 unsigned PPCInstrInfo::getSpillTarget() const {
3428 // With P10, we may need to spill paired vector registers or accumulator
3429 // registers. MMA implies paired vectors, so we can just check that.
3430 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3431 return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3432 2 : Subtarget.hasP9Vector() ?
3433 1 : 0;
3434 }
3435
getStoreOpcodesForSpillArray() const3436 ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3437 return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3438 }
3439
getLoadOpcodesForSpillArray() const3440 ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3441 return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3442 }
3443
3444 // This opt tries to convert the following imm form to an index form to save an
3445 // add for stack variables.
3446 // Return false if no such pattern found.
3447 //
3448 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3449 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3450 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3451 //
3452 // can be converted to:
3453 //
3454 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3455 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3456 //
3457 // In order to eliminate ADD instr, make sure that:
3458 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3459 // new ADDI instr and ADDI can only take int16 Imm.
3460 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3461 // between ADDI and ADD instr since its original def in ADDI will be changed
3462 // in new ADDI instr. And also there should be no new def for it between
3463 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3464 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3465 // between ADD and Imm instr since ADD instr will be eliminated.
3466 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3467 // moved to Index instr.
foldFrameOffset(MachineInstr & MI) const3468 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
3469 MachineFunction *MF = MI.getParent()->getParent();
3470 MachineRegisterInfo *MRI = &MF->getRegInfo();
3471 bool PostRA = !MRI->isSSA();
3472 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3473 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3474 // frame base(OffsetAddi) are determined.
3475 if (!PostRA)
3476 return false;
3477 unsigned ToBeDeletedReg = 0;
3478 int64_t OffsetImm = 0;
3479 unsigned XFormOpcode = 0;
3480 ImmInstrInfo III;
3481
3482 // Check if Imm instr meets requirement.
3483 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3484 III))
3485 return false;
3486
3487 bool OtherIntermediateUse = false;
3488 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3489
3490 // Exit if there is other use between ADD and Imm instr or no def found.
3491 if (OtherIntermediateUse || !ADDMI)
3492 return false;
3493
3494 // Check if ADD instr meets requirement.
3495 if (!isADDInstrEligibleForFolding(*ADDMI))
3496 return false;
3497
3498 unsigned ScaleRegIdx = 0;
3499 int64_t OffsetAddi = 0;
3500 MachineInstr *ADDIMI = nullptr;
3501
3502 // Check if there is a valid ToBeChangedReg in ADDMI.
3503 // 1: It must be killed.
3504 // 2: Its definition must be a valid ADDIMI.
3505 // 3: It must satify int16 offset requirement.
3506 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3507 ScaleRegIdx = 2;
3508 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3509 ScaleRegIdx = 1;
3510 else
3511 return false;
3512
3513 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3514 Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3515 Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3516 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3517 MachineBasicBlock::iterator End) {
3518 for (auto It = ++Start; It != End; It++)
3519 if (It->modifiesRegister(Reg, &getRegisterInfo()))
3520 return true;
3521 return false;
3522 };
3523
3524 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3525 // treated as special zero when ScaleReg is R0/X0 register.
3526 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3527 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3528 return false;
3529
3530 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3531 // and Imm Instr.
3532 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3533 return false;
3534
3535 // Now start to do the transformation.
3536 LLVM_DEBUG(dbgs() << "Replace instruction: "
3537 << "\n");
3538 LLVM_DEBUG(ADDIMI->dump());
3539 LLVM_DEBUG(ADDMI->dump());
3540 LLVM_DEBUG(MI.dump());
3541 LLVM_DEBUG(dbgs() << "with: "
3542 << "\n");
3543
3544 // Update ADDI instr.
3545 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3546
3547 // Update Imm instr.
3548 MI.setDesc(get(XFormOpcode));
3549 MI.getOperand(III.ImmOpNo)
3550 .ChangeToRegister(ScaleReg, false, false,
3551 ADDMI->getOperand(ScaleRegIdx).isKill());
3552
3553 MI.getOperand(III.OpNoForForwarding)
3554 .ChangeToRegister(ToBeChangedReg, false, false, true);
3555
3556 // Eliminate ADD instr.
3557 ADDMI->eraseFromParent();
3558
3559 LLVM_DEBUG(ADDIMI->dump());
3560 LLVM_DEBUG(MI.dump());
3561
3562 return true;
3563 }
3564
isADDIInstrEligibleForFolding(MachineInstr & ADDIMI,int64_t & Imm) const3565 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
3566 int64_t &Imm) const {
3567 unsigned Opc = ADDIMI.getOpcode();
3568
3569 // Exit if the instruction is not ADDI.
3570 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3571 return false;
3572
3573 // The operand may not necessarily be an immediate - it could be a relocation.
3574 if (!ADDIMI.getOperand(2).isImm())
3575 return false;
3576
3577 Imm = ADDIMI.getOperand(2).getImm();
3578
3579 return true;
3580 }
3581
isADDInstrEligibleForFolding(MachineInstr & ADDMI) const3582 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
3583 unsigned Opc = ADDMI.getOpcode();
3584
3585 // Exit if the instruction is not ADD.
3586 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3587 }
3588
isImmInstrEligibleForFolding(MachineInstr & MI,unsigned & ToBeDeletedReg,unsigned & XFormOpcode,int64_t & OffsetImm,ImmInstrInfo & III) const3589 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
3590 unsigned &ToBeDeletedReg,
3591 unsigned &XFormOpcode,
3592 int64_t &OffsetImm,
3593 ImmInstrInfo &III) const {
3594 // Only handle load/store.
3595 if (!MI.mayLoadOrStore())
3596 return false;
3597
3598 unsigned Opc = MI.getOpcode();
3599
3600 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3601
3602 // Exit if instruction has no index form.
3603 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3604 return false;
3605
3606 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3607 if (!instrHasImmForm(XFormOpcode,
3608 PPC::isVFRegister(MI.getOperand(0).getReg()), III, true))
3609 return false;
3610
3611 if (!III.IsSummingOperands)
3612 return false;
3613
3614 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3615 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3616 // Only support imm operands, not relocation slots or others.
3617 if (!ImmOperand.isImm())
3618 return false;
3619
3620 assert(RegOperand.isReg() && "Instruction format is not right");
3621
3622 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3623 if (!RegOperand.isKill())
3624 return false;
3625
3626 ToBeDeletedReg = RegOperand.getReg();
3627 OffsetImm = ImmOperand.getImm();
3628
3629 return true;
3630 }
3631
isValidToBeChangedReg(MachineInstr * ADDMI,unsigned Index,MachineInstr * & ADDIMI,int64_t & OffsetAddi,int64_t OffsetImm) const3632 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
3633 MachineInstr *&ADDIMI,
3634 int64_t &OffsetAddi,
3635 int64_t OffsetImm) const {
3636 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3637 MachineOperand &MO = ADDMI->getOperand(Index);
3638
3639 if (!MO.isKill())
3640 return false;
3641
3642 bool OtherIntermediateUse = false;
3643
3644 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3645 // Currently handle only one "add + Imminstr" pair case, exit if other
3646 // intermediate use for ToBeChangedReg found.
3647 // TODO: handle the cases where there are other "add + Imminstr" pairs
3648 // with same offset in Imminstr which is like:
3649 //
3650 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3651 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3652 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3653 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3654 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3655 //
3656 // can be converted to:
3657 //
3658 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3659 // (OffsetAddi + OffsetImm)
3660 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3661 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3662
3663 if (OtherIntermediateUse || !ADDIMI)
3664 return false;
3665 // Check if ADDI instr meets requirement.
3666 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3667 return false;
3668
3669 if (isInt<16>(OffsetAddi + OffsetImm))
3670 return true;
3671 return false;
3672 }
3673
3674 // If this instruction has an immediate form and one of its operands is a
3675 // result of a load-immediate or an add-immediate, convert it to
3676 // the immediate form if the constant is in range.
convertToImmediateForm(MachineInstr & MI,SmallSet<Register,4> & RegsToUpdate,MachineInstr ** KilledDef) const3677 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
3678 SmallSet<Register, 4> &RegsToUpdate,
3679 MachineInstr **KilledDef) const {
3680 MachineFunction *MF = MI.getParent()->getParent();
3681 MachineRegisterInfo *MRI = &MF->getRegInfo();
3682 bool PostRA = !MRI->isSSA();
3683 bool SeenIntermediateUse = true;
3684 unsigned ForwardingOperand = ~0U;
3685 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3686 SeenIntermediateUse);
3687 if (!DefMI)
3688 return false;
3689 assert(ForwardingOperand < MI.getNumOperands() &&
3690 "The forwarding operand needs to be valid at this point");
3691 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3692 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3693 if (KilledDef && KillFwdDefMI)
3694 *KilledDef = DefMI;
3695
3696 // Conservatively add defs from DefMI and defs/uses from MI to the set of
3697 // registers that need their kill flags updated.
3698 for (const MachineOperand &MO : DefMI->operands())
3699 if (MO.isReg() && MO.isDef())
3700 RegsToUpdate.insert(MO.getReg());
3701 for (const MachineOperand &MO : MI.operands())
3702 if (MO.isReg())
3703 RegsToUpdate.insert(MO.getReg());
3704
3705 // If this is a imm instruction and its register operands is produced by ADDI,
3706 // put the imm into imm inst directly.
3707 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3708 PPC::INSTRUCTION_LIST_END &&
3709 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3710 return true;
3711
3712 ImmInstrInfo III;
3713 bool IsVFReg = MI.getOperand(0).isReg()
3714 ? PPC::isVFRegister(MI.getOperand(0).getReg())
3715 : false;
3716 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3717 // If this is a reg+reg instruction that has a reg+imm form,
3718 // and one of the operands is produced by an add-immediate,
3719 // try to convert it.
3720 if (HasImmForm &&
3721 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3722 KillFwdDefMI))
3723 return true;
3724
3725 // If this is a reg+reg instruction that has a reg+imm form,
3726 // and one of the operands is produced by LI, convert it now.
3727 if (HasImmForm &&
3728 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3729 return true;
3730
3731 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3732 // can be simpified to LI.
3733 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3734 return true;
3735
3736 return false;
3737 }
3738
combineRLWINM(MachineInstr & MI,MachineInstr ** ToErase) const3739 bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
3740 MachineInstr **ToErase) const {
3741 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3742 Register FoldingReg = MI.getOperand(1).getReg();
3743 if (!FoldingReg.isVirtual())
3744 return false;
3745 MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3746 if (SrcMI->getOpcode() != PPC::RLWINM &&
3747 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3748 SrcMI->getOpcode() != PPC::RLWINM8 &&
3749 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3750 return false;
3751 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3752 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3753 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3754 "Invalid PPC::RLWINM Instruction!");
3755 uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3756 uint64_t SHMI = MI.getOperand(2).getImm();
3757 uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3758 uint64_t MBMI = MI.getOperand(3).getImm();
3759 uint64_t MESrc = SrcMI->getOperand(4).getImm();
3760 uint64_t MEMI = MI.getOperand(4).getImm();
3761
3762 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3763 "Invalid PPC::RLWINM Instruction!");
3764 // If MBMI is bigger than MEMI, we always can not get run of ones.
3765 // RotatedSrcMask non-wrap:
3766 // 0........31|32........63
3767 // RotatedSrcMask: B---E B---E
3768 // MaskMI: -----------|--E B------
3769 // Result: ----- --- (Bad candidate)
3770 //
3771 // RotatedSrcMask wrap:
3772 // 0........31|32........63
3773 // RotatedSrcMask: --E B----|--E B----
3774 // MaskMI: -----------|--E B------
3775 // Result: --- -----|--- ----- (Bad candidate)
3776 //
3777 // One special case is RotatedSrcMask is a full set mask.
3778 // RotatedSrcMask full:
3779 // 0........31|32........63
3780 // RotatedSrcMask: ------EB---|-------EB---
3781 // MaskMI: -----------|--E B------
3782 // Result: -----------|--- ------- (Good candidate)
3783
3784 // Mark special case.
3785 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3786
3787 // For other MBMI > MEMI cases, just return.
3788 if ((MBMI > MEMI) && !SrcMaskFull)
3789 return false;
3790
3791 // Handle MBMI <= MEMI cases.
3792 APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3793 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3794 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3795 // while in PowerPC ISA, lowerest bit is at index 63.
3796 APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3797
3798 APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3799 APInt FinalMask = RotatedSrcMask & MaskMI;
3800 uint32_t NewMB, NewME;
3801 bool Simplified = false;
3802
3803 // If final mask is 0, MI result should be 0 too.
3804 if (FinalMask.isZero()) {
3805 bool Is64Bit =
3806 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3807 Simplified = true;
3808 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3809 LLVM_DEBUG(MI.dump());
3810
3811 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3812 // Replace MI with "LI 0"
3813 MI.removeOperand(4);
3814 MI.removeOperand(3);
3815 MI.removeOperand(2);
3816 MI.getOperand(1).ChangeToImmediate(0);
3817 MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3818 } else {
3819 // Replace MI with "ANDI_rec reg, 0"
3820 MI.removeOperand(4);
3821 MI.removeOperand(3);
3822 MI.getOperand(2).setImm(0);
3823 MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3824 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3825 if (SrcMI->getOperand(1).isKill()) {
3826 MI.getOperand(1).setIsKill(true);
3827 SrcMI->getOperand(1).setIsKill(false);
3828 } else
3829 // About to replace MI.getOperand(1), clear its kill flag.
3830 MI.getOperand(1).setIsKill(false);
3831 }
3832
3833 LLVM_DEBUG(dbgs() << "With: ");
3834 LLVM_DEBUG(MI.dump());
3835
3836 } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3837 NewMB <= NewME) ||
3838 SrcMaskFull) {
3839 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3840 // than NewME. Otherwise we get a 64 bit value after folding, but MI
3841 // return a 32 bit value.
3842 Simplified = true;
3843 LLVM_DEBUG(dbgs() << "Converting Instr: ");
3844 LLVM_DEBUG(MI.dump());
3845
3846 uint16_t NewSH = (SHSrc + SHMI) % 32;
3847 MI.getOperand(2).setImm(NewSH);
3848 // If SrcMI mask is full, no need to update MBMI and MEMI.
3849 if (!SrcMaskFull) {
3850 MI.getOperand(3).setImm(NewMB);
3851 MI.getOperand(4).setImm(NewME);
3852 }
3853 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3854 if (SrcMI->getOperand(1).isKill()) {
3855 MI.getOperand(1).setIsKill(true);
3856 SrcMI->getOperand(1).setIsKill(false);
3857 } else
3858 // About to replace MI.getOperand(1), clear its kill flag.
3859 MI.getOperand(1).setIsKill(false);
3860
3861 LLVM_DEBUG(dbgs() << "To: ");
3862 LLVM_DEBUG(MI.dump());
3863 }
3864 if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3865 !SrcMI->hasImplicitDef()) {
3866 // If FoldingReg has no non-debug use and it has no implicit def (it
3867 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3868 // Otherwise keep it.
3869 *ToErase = SrcMI;
3870 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3871 LLVM_DEBUG(SrcMI->dump());
3872 }
3873 return Simplified;
3874 }
3875
instrHasImmForm(unsigned Opc,bool IsVFReg,ImmInstrInfo & III,bool PostRA) const3876 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3877 ImmInstrInfo &III, bool PostRA) const {
3878 // The vast majority of the instructions would need their operand 2 replaced
3879 // with an immediate when switching to the reg+imm form. A marked exception
3880 // are the update form loads/stores for which a constant operand 2 would need
3881 // to turn into a displacement and move operand 1 to the operand 2 position.
3882 III.ImmOpNo = 2;
3883 III.OpNoForForwarding = 2;
3884 III.ImmWidth = 16;
3885 III.ImmMustBeMultipleOf = 1;
3886 III.TruncateImmTo = 0;
3887 III.IsSummingOperands = false;
3888 switch (Opc) {
3889 default: return false;
3890 case PPC::ADD4:
3891 case PPC::ADD8:
3892 III.SignedImm = true;
3893 III.ZeroIsSpecialOrig = 0;
3894 III.ZeroIsSpecialNew = 1;
3895 III.IsCommutative = true;
3896 III.IsSummingOperands = true;
3897 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3898 break;
3899 case PPC::ADDC:
3900 case PPC::ADDC8:
3901 III.SignedImm = true;
3902 III.ZeroIsSpecialOrig = 0;
3903 III.ZeroIsSpecialNew = 0;
3904 III.IsCommutative = true;
3905 III.IsSummingOperands = true;
3906 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3907 break;
3908 case PPC::ADDC_rec:
3909 III.SignedImm = true;
3910 III.ZeroIsSpecialOrig = 0;
3911 III.ZeroIsSpecialNew = 0;
3912 III.IsCommutative = true;
3913 III.IsSummingOperands = true;
3914 III.ImmOpcode = PPC::ADDIC_rec;
3915 break;
3916 case PPC::SUBFC:
3917 case PPC::SUBFC8:
3918 III.SignedImm = true;
3919 III.ZeroIsSpecialOrig = 0;
3920 III.ZeroIsSpecialNew = 0;
3921 III.IsCommutative = false;
3922 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3923 break;
3924 case PPC::CMPW:
3925 case PPC::CMPD:
3926 III.SignedImm = true;
3927 III.ZeroIsSpecialOrig = 0;
3928 III.ZeroIsSpecialNew = 0;
3929 III.IsCommutative = false;
3930 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3931 break;
3932 case PPC::CMPLW:
3933 case PPC::CMPLD:
3934 III.SignedImm = false;
3935 III.ZeroIsSpecialOrig = 0;
3936 III.ZeroIsSpecialNew = 0;
3937 III.IsCommutative = false;
3938 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3939 break;
3940 case PPC::AND_rec:
3941 case PPC::AND8_rec:
3942 case PPC::OR:
3943 case PPC::OR8:
3944 case PPC::XOR:
3945 case PPC::XOR8:
3946 III.SignedImm = false;
3947 III.ZeroIsSpecialOrig = 0;
3948 III.ZeroIsSpecialNew = 0;
3949 III.IsCommutative = true;
3950 switch(Opc) {
3951 default: llvm_unreachable("Unknown opcode");
3952 case PPC::AND_rec:
3953 III.ImmOpcode = PPC::ANDI_rec;
3954 break;
3955 case PPC::AND8_rec:
3956 III.ImmOpcode = PPC::ANDI8_rec;
3957 break;
3958 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3959 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3960 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3961 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3962 }
3963 break;
3964 case PPC::RLWNM:
3965 case PPC::RLWNM8:
3966 case PPC::RLWNM_rec:
3967 case PPC::RLWNM8_rec:
3968 case PPC::SLW:
3969 case PPC::SLW8:
3970 case PPC::SLW_rec:
3971 case PPC::SLW8_rec:
3972 case PPC::SRW:
3973 case PPC::SRW8:
3974 case PPC::SRW_rec:
3975 case PPC::SRW8_rec:
3976 case PPC::SRAW:
3977 case PPC::SRAW_rec:
3978 III.SignedImm = false;
3979 III.ZeroIsSpecialOrig = 0;
3980 III.ZeroIsSpecialNew = 0;
3981 III.IsCommutative = false;
3982 // This isn't actually true, but the instructions ignore any of the
3983 // upper bits, so any immediate loaded with an LI is acceptable.
3984 // This does not apply to shift right algebraic because a value
3985 // out of range will produce a -1/0.
3986 III.ImmWidth = 16;
3987 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3988 Opc == PPC::RLWNM8_rec)
3989 III.TruncateImmTo = 5;
3990 else
3991 III.TruncateImmTo = 6;
3992 switch(Opc) {
3993 default: llvm_unreachable("Unknown opcode");
3994 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3995 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3996 case PPC::RLWNM_rec:
3997 III.ImmOpcode = PPC::RLWINM_rec;
3998 break;
3999 case PPC::RLWNM8_rec:
4000 III.ImmOpcode = PPC::RLWINM8_rec;
4001 break;
4002 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4003 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4004 case PPC::SLW_rec:
4005 III.ImmOpcode = PPC::RLWINM_rec;
4006 break;
4007 case PPC::SLW8_rec:
4008 III.ImmOpcode = PPC::RLWINM8_rec;
4009 break;
4010 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4011 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4012 case PPC::SRW_rec:
4013 III.ImmOpcode = PPC::RLWINM_rec;
4014 break;
4015 case PPC::SRW8_rec:
4016 III.ImmOpcode = PPC::RLWINM8_rec;
4017 break;
4018 case PPC::SRAW:
4019 III.ImmWidth = 5;
4020 III.TruncateImmTo = 0;
4021 III.ImmOpcode = PPC::SRAWI;
4022 break;
4023 case PPC::SRAW_rec:
4024 III.ImmWidth = 5;
4025 III.TruncateImmTo = 0;
4026 III.ImmOpcode = PPC::SRAWI_rec;
4027 break;
4028 }
4029 break;
4030 case PPC::RLDCL:
4031 case PPC::RLDCL_rec:
4032 case PPC::RLDCR:
4033 case PPC::RLDCR_rec:
4034 case PPC::SLD:
4035 case PPC::SLD_rec:
4036 case PPC::SRD:
4037 case PPC::SRD_rec:
4038 case PPC::SRAD:
4039 case PPC::SRAD_rec:
4040 III.SignedImm = false;
4041 III.ZeroIsSpecialOrig = 0;
4042 III.ZeroIsSpecialNew = 0;
4043 III.IsCommutative = false;
4044 // This isn't actually true, but the instructions ignore any of the
4045 // upper bits, so any immediate loaded with an LI is acceptable.
4046 // This does not apply to shift right algebraic because a value
4047 // out of range will produce a -1/0.
4048 III.ImmWidth = 16;
4049 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4050 Opc == PPC::RLDCR_rec)
4051 III.TruncateImmTo = 6;
4052 else
4053 III.TruncateImmTo = 7;
4054 switch(Opc) {
4055 default: llvm_unreachable("Unknown opcode");
4056 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4057 case PPC::RLDCL_rec:
4058 III.ImmOpcode = PPC::RLDICL_rec;
4059 break;
4060 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4061 case PPC::RLDCR_rec:
4062 III.ImmOpcode = PPC::RLDICR_rec;
4063 break;
4064 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4065 case PPC::SLD_rec:
4066 III.ImmOpcode = PPC::RLDICR_rec;
4067 break;
4068 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4069 case PPC::SRD_rec:
4070 III.ImmOpcode = PPC::RLDICL_rec;
4071 break;
4072 case PPC::SRAD:
4073 III.ImmWidth = 6;
4074 III.TruncateImmTo = 0;
4075 III.ImmOpcode = PPC::SRADI;
4076 break;
4077 case PPC::SRAD_rec:
4078 III.ImmWidth = 6;
4079 III.TruncateImmTo = 0;
4080 III.ImmOpcode = PPC::SRADI_rec;
4081 break;
4082 }
4083 break;
4084 // Loads and stores:
4085 case PPC::LBZX:
4086 case PPC::LBZX8:
4087 case PPC::LHZX:
4088 case PPC::LHZX8:
4089 case PPC::LHAX:
4090 case PPC::LHAX8:
4091 case PPC::LWZX:
4092 case PPC::LWZX8:
4093 case PPC::LWAX:
4094 case PPC::LDX:
4095 case PPC::LFSX:
4096 case PPC::LFDX:
4097 case PPC::STBX:
4098 case PPC::STBX8:
4099 case PPC::STHX:
4100 case PPC::STHX8:
4101 case PPC::STWX:
4102 case PPC::STWX8:
4103 case PPC::STDX:
4104 case PPC::STFSX:
4105 case PPC::STFDX:
4106 III.SignedImm = true;
4107 III.ZeroIsSpecialOrig = 1;
4108 III.ZeroIsSpecialNew = 2;
4109 III.IsCommutative = true;
4110 III.IsSummingOperands = true;
4111 III.ImmOpNo = 1;
4112 III.OpNoForForwarding = 2;
4113 switch(Opc) {
4114 default: llvm_unreachable("Unknown opcode");
4115 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4116 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4117 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4118 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4119 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4120 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4121 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4122 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4123 case PPC::LWAX:
4124 III.ImmOpcode = PPC::LWA;
4125 III.ImmMustBeMultipleOf = 4;
4126 break;
4127 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4128 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4129 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4130 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4131 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4132 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4133 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4134 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4135 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4136 case PPC::STDX:
4137 III.ImmOpcode = PPC::STD;
4138 III.ImmMustBeMultipleOf = 4;
4139 break;
4140 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4141 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4142 }
4143 break;
4144 case PPC::LBZUX:
4145 case PPC::LBZUX8:
4146 case PPC::LHZUX:
4147 case PPC::LHZUX8:
4148 case PPC::LHAUX:
4149 case PPC::LHAUX8:
4150 case PPC::LWZUX:
4151 case PPC::LWZUX8:
4152 case PPC::LDUX:
4153 case PPC::LFSUX:
4154 case PPC::LFDUX:
4155 case PPC::STBUX:
4156 case PPC::STBUX8:
4157 case PPC::STHUX:
4158 case PPC::STHUX8:
4159 case PPC::STWUX:
4160 case PPC::STWUX8:
4161 case PPC::STDUX:
4162 case PPC::STFSUX:
4163 case PPC::STFDUX:
4164 III.SignedImm = true;
4165 III.ZeroIsSpecialOrig = 2;
4166 III.ZeroIsSpecialNew = 3;
4167 III.IsCommutative = false;
4168 III.IsSummingOperands = true;
4169 III.ImmOpNo = 2;
4170 III.OpNoForForwarding = 3;
4171 switch(Opc) {
4172 default: llvm_unreachable("Unknown opcode");
4173 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4174 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4175 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4176 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4177 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4178 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4179 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4180 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4181 case PPC::LDUX:
4182 III.ImmOpcode = PPC::LDU;
4183 III.ImmMustBeMultipleOf = 4;
4184 break;
4185 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4186 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4187 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4188 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4189 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4190 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4191 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4192 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4193 case PPC::STDUX:
4194 III.ImmOpcode = PPC::STDU;
4195 III.ImmMustBeMultipleOf = 4;
4196 break;
4197 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4198 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4199 }
4200 break;
4201 // Power9 and up only. For some of these, the X-Form version has access to all
4202 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4203 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4204 // into or stored from is one of the VR registers.
4205 case PPC::LXVX:
4206 case PPC::LXSSPX:
4207 case PPC::LXSDX:
4208 case PPC::STXVX:
4209 case PPC::STXSSPX:
4210 case PPC::STXSDX:
4211 case PPC::XFLOADf32:
4212 case PPC::XFLOADf64:
4213 case PPC::XFSTOREf32:
4214 case PPC::XFSTOREf64:
4215 if (!Subtarget.hasP9Vector())
4216 return false;
4217 III.SignedImm = true;
4218 III.ZeroIsSpecialOrig = 1;
4219 III.ZeroIsSpecialNew = 2;
4220 III.IsCommutative = true;
4221 III.IsSummingOperands = true;
4222 III.ImmOpNo = 1;
4223 III.OpNoForForwarding = 2;
4224 III.ImmMustBeMultipleOf = 4;
4225 switch(Opc) {
4226 default: llvm_unreachable("Unknown opcode");
4227 case PPC::LXVX:
4228 III.ImmOpcode = PPC::LXV;
4229 III.ImmMustBeMultipleOf = 16;
4230 break;
4231 case PPC::LXSSPX:
4232 if (PostRA) {
4233 if (IsVFReg)
4234 III.ImmOpcode = PPC::LXSSP;
4235 else {
4236 III.ImmOpcode = PPC::LFS;
4237 III.ImmMustBeMultipleOf = 1;
4238 }
4239 break;
4240 }
4241 [[fallthrough]];
4242 case PPC::XFLOADf32:
4243 III.ImmOpcode = PPC::DFLOADf32;
4244 break;
4245 case PPC::LXSDX:
4246 if (PostRA) {
4247 if (IsVFReg)
4248 III.ImmOpcode = PPC::LXSD;
4249 else {
4250 III.ImmOpcode = PPC::LFD;
4251 III.ImmMustBeMultipleOf = 1;
4252 }
4253 break;
4254 }
4255 [[fallthrough]];
4256 case PPC::XFLOADf64:
4257 III.ImmOpcode = PPC::DFLOADf64;
4258 break;
4259 case PPC::STXVX:
4260 III.ImmOpcode = PPC::STXV;
4261 III.ImmMustBeMultipleOf = 16;
4262 break;
4263 case PPC::STXSSPX:
4264 if (PostRA) {
4265 if (IsVFReg)
4266 III.ImmOpcode = PPC::STXSSP;
4267 else {
4268 III.ImmOpcode = PPC::STFS;
4269 III.ImmMustBeMultipleOf = 1;
4270 }
4271 break;
4272 }
4273 [[fallthrough]];
4274 case PPC::XFSTOREf32:
4275 III.ImmOpcode = PPC::DFSTOREf32;
4276 break;
4277 case PPC::STXSDX:
4278 if (PostRA) {
4279 if (IsVFReg)
4280 III.ImmOpcode = PPC::STXSD;
4281 else {
4282 III.ImmOpcode = PPC::STFD;
4283 III.ImmMustBeMultipleOf = 1;
4284 }
4285 break;
4286 }
4287 [[fallthrough]];
4288 case PPC::XFSTOREf64:
4289 III.ImmOpcode = PPC::DFSTOREf64;
4290 break;
4291 }
4292 break;
4293 }
4294 return true;
4295 }
4296
4297 // Utility function for swaping two arbitrary operands of an instruction.
swapMIOperands(MachineInstr & MI,unsigned Op1,unsigned Op2)4298 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4299 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4300
4301 unsigned MaxOp = std::max(Op1, Op2);
4302 unsigned MinOp = std::min(Op1, Op2);
4303 MachineOperand MOp1 = MI.getOperand(MinOp);
4304 MachineOperand MOp2 = MI.getOperand(MaxOp);
4305 MI.removeOperand(std::max(Op1, Op2));
4306 MI.removeOperand(std::min(Op1, Op2));
4307
4308 // If the operands we are swapping are the two at the end (the common case)
4309 // we can just remove both and add them in the opposite order.
4310 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4311 MI.addOperand(MOp2);
4312 MI.addOperand(MOp1);
4313 } else {
4314 // Store all operands in a temporary vector, remove them and re-add in the
4315 // right order.
4316 SmallVector<MachineOperand, 2> MOps;
4317 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4318 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4319 MOps.push_back(MI.getOperand(i));
4320 MI.removeOperand(i);
4321 }
4322 // MOp2 needs to be added next.
4323 MI.addOperand(MOp2);
4324 // Now add the rest.
4325 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4326 if (i == MaxOp)
4327 MI.addOperand(MOp1);
4328 else {
4329 MI.addOperand(MOps.back());
4330 MOps.pop_back();
4331 }
4332 }
4333 }
4334 }
4335
4336 // Check if the 'MI' that has the index OpNoForForwarding
4337 // meets the requirement described in the ImmInstrInfo.
isUseMIElgibleForForwarding(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding) const4338 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4339 const ImmInstrInfo &III,
4340 unsigned OpNoForForwarding
4341 ) const {
4342 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4343 // would not work pre-RA, we can only do the check post RA.
4344 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4345 if (MRI.isSSA())
4346 return false;
4347
4348 // Cannot do the transform if MI isn't summing the operands.
4349 if (!III.IsSummingOperands)
4350 return false;
4351
4352 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4353 if (!III.ZeroIsSpecialOrig)
4354 return false;
4355
4356 // We cannot do the transform if the operand we are trying to replace
4357 // isn't the same as the operand the instruction allows.
4358 if (OpNoForForwarding != III.OpNoForForwarding)
4359 return false;
4360
4361 // Check if the instruction we are trying to transform really has
4362 // the special zero register as its operand.
4363 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4364 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4365 return false;
4366
4367 // This machine instruction is convertible if it is,
4368 // 1. summing the operands.
4369 // 2. one of the operands is special zero register.
4370 // 3. the operand we are trying to replace is allowed by the MI.
4371 return true;
4372 }
4373
4374 // Check if the DefMI is the add inst and set the ImmMO and RegMO
4375 // accordingly.
isDefMIElgibleForForwarding(MachineInstr & DefMI,const ImmInstrInfo & III,MachineOperand * & ImmMO,MachineOperand * & RegMO) const4376 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4377 const ImmInstrInfo &III,
4378 MachineOperand *&ImmMO,
4379 MachineOperand *&RegMO) const {
4380 unsigned Opc = DefMI.getOpcode();
4381 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4382 return false;
4383
4384 assert(DefMI.getNumOperands() >= 3 &&
4385 "Add inst must have at least three operands");
4386 RegMO = &DefMI.getOperand(1);
4387 ImmMO = &DefMI.getOperand(2);
4388
4389 // Before RA, ADDI first operand could be a frame index.
4390 if (!RegMO->isReg())
4391 return false;
4392
4393 // This DefMI is elgible for forwarding if it is:
4394 // 1. add inst
4395 // 2. one of the operands is Imm/CPI/Global.
4396 return isAnImmediateOperand(*ImmMO);
4397 }
4398
isRegElgibleForForwarding(const MachineOperand & RegMO,const MachineInstr & DefMI,const MachineInstr & MI,bool KillDefMI,bool & IsFwdFeederRegKilled,bool & SeenIntermediateUse) const4399 bool PPCInstrInfo::isRegElgibleForForwarding(
4400 const MachineOperand &RegMO, const MachineInstr &DefMI,
4401 const MachineInstr &MI, bool KillDefMI,
4402 bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4403 // x = addi y, imm
4404 // ...
4405 // z = lfdx 0, x -> z = lfd imm(y)
4406 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4407 // of "y" between the DEF of "x" and "z".
4408 // The query is only valid post RA.
4409 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4410 if (MRI.isSSA())
4411 return false;
4412
4413 Register Reg = RegMO.getReg();
4414
4415 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4416 MachineBasicBlock::const_reverse_iterator It = MI;
4417 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4418 It++;
4419 for (; It != E; ++It) {
4420 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4421 return false;
4422 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4423 IsFwdFeederRegKilled = true;
4424 if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4425 SeenIntermediateUse = true;
4426 // Made it to DefMI without encountering a clobber.
4427 if ((&*It) == &DefMI)
4428 break;
4429 }
4430 assert((&*It) == &DefMI && "DefMI is missing");
4431
4432 // If DefMI also defines the register to be forwarded, we can only forward it
4433 // if DefMI is being erased.
4434 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4435 return KillDefMI;
4436
4437 return true;
4438 }
4439
isImmElgibleForForwarding(const MachineOperand & ImmMO,const MachineInstr & DefMI,const ImmInstrInfo & III,int64_t & Imm,int64_t BaseImm) const4440 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4441 const MachineInstr &DefMI,
4442 const ImmInstrInfo &III,
4443 int64_t &Imm,
4444 int64_t BaseImm) const {
4445 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4446 if (DefMI.getOpcode() == PPC::ADDItocL) {
4447 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4448 // However, we know that, it is 16-bit width, and has the alignment of 4.
4449 // Check if the instruction met the requirement.
4450 if (III.ImmMustBeMultipleOf > 4 ||
4451 III.TruncateImmTo || III.ImmWidth != 16)
4452 return false;
4453
4454 // Going from XForm to DForm loads means that the displacement needs to be
4455 // not just an immediate but also a multiple of 4, or 16 depending on the
4456 // load. A DForm load cannot be represented if it is a multiple of say 2.
4457 // XForm loads do not have this restriction.
4458 if (ImmMO.isGlobal()) {
4459 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4460 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
4461 return false;
4462 }
4463
4464 return true;
4465 }
4466
4467 if (ImmMO.isImm()) {
4468 // It is Imm, we need to check if the Imm fit the range.
4469 // Sign-extend to 64-bits.
4470 // DefMI may be folded with another imm form instruction, the result Imm is
4471 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4472 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4473 if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4474 return false;
4475 if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4476 return false;
4477 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4478
4479 if (Imm % III.ImmMustBeMultipleOf)
4480 return false;
4481 if (III.TruncateImmTo)
4482 Imm &= ((1 << III.TruncateImmTo) - 1);
4483 }
4484 else
4485 return false;
4486
4487 // This ImmMO is forwarded if it meets the requriement describle
4488 // in ImmInstrInfo
4489 return true;
4490 }
4491
simplifyToLI(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding,MachineInstr ** KilledDef) const4492 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4493 unsigned OpNoForForwarding,
4494 MachineInstr **KilledDef) const {
4495 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4496 !DefMI.getOperand(1).isImm())
4497 return false;
4498
4499 MachineFunction *MF = MI.getParent()->getParent();
4500 MachineRegisterInfo *MRI = &MF->getRegInfo();
4501 bool PostRA = !MRI->isSSA();
4502
4503 int64_t Immediate = DefMI.getOperand(1).getImm();
4504 // Sign-extend to 64-bits.
4505 int64_t SExtImm = SignExtend64<16>(Immediate);
4506
4507 bool ReplaceWithLI = false;
4508 bool Is64BitLI = false;
4509 int64_t NewImm = 0;
4510 bool SetCR = false;
4511 unsigned Opc = MI.getOpcode();
4512 switch (Opc) {
4513 default:
4514 return false;
4515
4516 // FIXME: Any branches conditional on such a comparison can be made
4517 // unconditional. At this time, this happens too infrequently to be worth
4518 // the implementation effort, but if that ever changes, we could convert
4519 // such a pattern here.
4520 case PPC::CMPWI:
4521 case PPC::CMPLWI:
4522 case PPC::CMPDI:
4523 case PPC::CMPLDI: {
4524 // Doing this post-RA would require dataflow analysis to reliably find uses
4525 // of the CR register set by the compare.
4526 // No need to fixup killed/dead flag since this transformation is only valid
4527 // before RA.
4528 if (PostRA)
4529 return false;
4530 // If a compare-immediate is fed by an immediate and is itself an input of
4531 // an ISEL (the most common case) into a COPY of the correct register.
4532 bool Changed = false;
4533 Register DefReg = MI.getOperand(0).getReg();
4534 int64_t Comparand = MI.getOperand(2).getImm();
4535 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4536 ? (Comparand | 0xFFFFFFFFFFFF0000)
4537 : Comparand;
4538
4539 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4540 unsigned UseOpc = CompareUseMI.getOpcode();
4541 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4542 continue;
4543 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4544 Register TrueReg = CompareUseMI.getOperand(1).getReg();
4545 Register FalseReg = CompareUseMI.getOperand(2).getReg();
4546 unsigned RegToCopy =
4547 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4548 if (RegToCopy == PPC::NoRegister)
4549 continue;
4550 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4551 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4552 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4553 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4554 CompareUseMI.removeOperand(3);
4555 CompareUseMI.removeOperand(2);
4556 continue;
4557 }
4558 LLVM_DEBUG(
4559 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4560 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4561 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4562 // Convert to copy and remove unneeded operands.
4563 CompareUseMI.setDesc(get(PPC::COPY));
4564 CompareUseMI.removeOperand(3);
4565 CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4566 CmpIselsConverted++;
4567 Changed = true;
4568 LLVM_DEBUG(CompareUseMI.dump());
4569 }
4570 if (Changed)
4571 return true;
4572 // This may end up incremented multiple times since this function is called
4573 // during a fixed-point transformation, but it is only meant to indicate the
4574 // presence of this opportunity.
4575 MissedConvertibleImmediateInstrs++;
4576 return false;
4577 }
4578
4579 // Immediate forms - may simply be convertable to an LI.
4580 case PPC::ADDI:
4581 case PPC::ADDI8: {
4582 // Does the sum fit in a 16-bit signed field?
4583 int64_t Addend = MI.getOperand(2).getImm();
4584 if (isInt<16>(Addend + SExtImm)) {
4585 ReplaceWithLI = true;
4586 Is64BitLI = Opc == PPC::ADDI8;
4587 NewImm = Addend + SExtImm;
4588 break;
4589 }
4590 return false;
4591 }
4592 case PPC::SUBFIC:
4593 case PPC::SUBFIC8: {
4594 // Only transform this if the CARRY implicit operand is dead.
4595 if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4596 return false;
4597 int64_t Minuend = MI.getOperand(2).getImm();
4598 if (isInt<16>(Minuend - SExtImm)) {
4599 ReplaceWithLI = true;
4600 Is64BitLI = Opc == PPC::SUBFIC8;
4601 NewImm = Minuend - SExtImm;
4602 break;
4603 }
4604 return false;
4605 }
4606 case PPC::RLDICL:
4607 case PPC::RLDICL_rec:
4608 case PPC::RLDICL_32:
4609 case PPC::RLDICL_32_64: {
4610 // Use APInt's rotate function.
4611 int64_t SH = MI.getOperand(2).getImm();
4612 int64_t MB = MI.getOperand(3).getImm();
4613 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4614 SExtImm, true);
4615 InVal = InVal.rotl(SH);
4616 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4617 InVal &= Mask;
4618 // Can't replace negative values with an LI as that will sign-extend
4619 // and not clear the left bits. If we're setting the CR bit, we will use
4620 // ANDI_rec which won't sign extend, so that's safe.
4621 if (isUInt<15>(InVal.getSExtValue()) ||
4622 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4623 ReplaceWithLI = true;
4624 Is64BitLI = Opc != PPC::RLDICL_32;
4625 NewImm = InVal.getSExtValue();
4626 SetCR = Opc == PPC::RLDICL_rec;
4627 break;
4628 }
4629 return false;
4630 }
4631 case PPC::RLWINM:
4632 case PPC::RLWINM8:
4633 case PPC::RLWINM_rec:
4634 case PPC::RLWINM8_rec: {
4635 int64_t SH = MI.getOperand(2).getImm();
4636 int64_t MB = MI.getOperand(3).getImm();
4637 int64_t ME = MI.getOperand(4).getImm();
4638 APInt InVal(32, SExtImm, true);
4639 InVal = InVal.rotl(SH);
4640 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4641 InVal &= Mask;
4642 // Can't replace negative values with an LI as that will sign-extend
4643 // and not clear the left bits. If we're setting the CR bit, we will use
4644 // ANDI_rec which won't sign extend, so that's safe.
4645 bool ValueFits = isUInt<15>(InVal.getSExtValue());
4646 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4647 isUInt<16>(InVal.getSExtValue()));
4648 if (ValueFits) {
4649 ReplaceWithLI = true;
4650 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4651 NewImm = InVal.getSExtValue();
4652 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4653 break;
4654 }
4655 return false;
4656 }
4657 case PPC::ORI:
4658 case PPC::ORI8:
4659 case PPC::XORI:
4660 case PPC::XORI8: {
4661 int64_t LogicalImm = MI.getOperand(2).getImm();
4662 int64_t Result = 0;
4663 if (Opc == PPC::ORI || Opc == PPC::ORI8)
4664 Result = LogicalImm | SExtImm;
4665 else
4666 Result = LogicalImm ^ SExtImm;
4667 if (isInt<16>(Result)) {
4668 ReplaceWithLI = true;
4669 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4670 NewImm = Result;
4671 break;
4672 }
4673 return false;
4674 }
4675 }
4676
4677 if (ReplaceWithLI) {
4678 // We need to be careful with CR-setting instructions we're replacing.
4679 if (SetCR) {
4680 // We don't know anything about uses when we're out of SSA, so only
4681 // replace if the new immediate will be reproduced.
4682 bool ImmChanged = (SExtImm & NewImm) != NewImm;
4683 if (PostRA && ImmChanged)
4684 return false;
4685
4686 if (!PostRA) {
4687 // If the defining load-immediate has no other uses, we can just replace
4688 // the immediate with the new immediate.
4689 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4690 DefMI.getOperand(1).setImm(NewImm);
4691
4692 // If we're not using the GPR result of the CR-setting instruction, we
4693 // just need to and with zero/non-zero depending on the new immediate.
4694 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
4695 if (NewImm) {
4696 assert(Immediate && "Transformation converted zero to non-zero?");
4697 NewImm = Immediate;
4698 }
4699 } else if (ImmChanged)
4700 return false;
4701 }
4702 }
4703
4704 LLVM_DEBUG(dbgs() << "Replacing constant instruction:\n");
4705 LLVM_DEBUG(MI.dump());
4706 LLVM_DEBUG(dbgs() << "Fed by:\n");
4707 LLVM_DEBUG(DefMI.dump());
4708 LoadImmediateInfo LII;
4709 LII.Imm = NewImm;
4710 LII.Is64Bit = Is64BitLI;
4711 LII.SetCR = SetCR;
4712 // If we're setting the CR, the original load-immediate must be kept (as an
4713 // operand to ANDI_rec/ANDI8_rec).
4714 if (KilledDef && SetCR)
4715 *KilledDef = nullptr;
4716 replaceInstrWithLI(MI, LII);
4717
4718 if (PostRA)
4719 recomputeLivenessFlags(*MI.getParent());
4720
4721 LLVM_DEBUG(dbgs() << "With:\n");
4722 LLVM_DEBUG(MI.dump());
4723 return true;
4724 }
4725 return false;
4726 }
4727
transformToNewImmFormFedByAdd(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding) const4728 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4729 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4730 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4731 bool PostRA = !MRI->isSSA();
4732 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4733 // for post-ra.
4734 if (PostRA)
4735 return false;
4736
4737 // Only handle load/store.
4738 if (!MI.mayLoadOrStore())
4739 return false;
4740
4741 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
4742
4743 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4744 "MI must have x-form opcode");
4745
4746 // get Imm Form info.
4747 ImmInstrInfo III;
4748 bool IsVFReg = MI.getOperand(0).isReg()
4749 ? PPC::isVFRegister(MI.getOperand(0).getReg())
4750 : false;
4751
4752 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
4753 return false;
4754
4755 if (!III.IsSummingOperands)
4756 return false;
4757
4758 if (OpNoForForwarding != III.OpNoForForwarding)
4759 return false;
4760
4761 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
4762 if (!ImmOperandMI.isImm())
4763 return false;
4764
4765 // Check DefMI.
4766 MachineOperand *ImmMO = nullptr;
4767 MachineOperand *RegMO = nullptr;
4768 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4769 return false;
4770 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4771
4772 // Check Imm.
4773 // Set ImmBase from imm instruction as base and get new Imm inside
4774 // isImmElgibleForForwarding.
4775 int64_t ImmBase = ImmOperandMI.getImm();
4776 int64_t Imm = 0;
4777 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
4778 return false;
4779
4780 // Do the transform
4781 LLVM_DEBUG(dbgs() << "Replacing existing reg+imm instruction:\n");
4782 LLVM_DEBUG(MI.dump());
4783 LLVM_DEBUG(dbgs() << "Fed by:\n");
4784 LLVM_DEBUG(DefMI.dump());
4785
4786 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
4787 MI.getOperand(III.ImmOpNo).setImm(Imm);
4788
4789 LLVM_DEBUG(dbgs() << "With:\n");
4790 LLVM_DEBUG(MI.dump());
4791 return true;
4792 }
4793
4794 // If an X-Form instruction is fed by an add-immediate and one of its operands
4795 // is the literal zero, attempt to forward the source of the add-immediate to
4796 // the corresponding D-Form instruction with the displacement coming from
4797 // the immediate being added.
transformToImmFormFedByAdd(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding,MachineInstr & DefMI,bool KillDefMI) const4798 bool PPCInstrInfo::transformToImmFormFedByAdd(
4799 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4800 MachineInstr &DefMI, bool KillDefMI) const {
4801 // RegMO ImmMO
4802 // | |
4803 // x = addi reg, imm <----- DefMI
4804 // y = op 0 , x <----- MI
4805 // |
4806 // OpNoForForwarding
4807 // Check if the MI meet the requirement described in the III.
4808 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
4809 return false;
4810
4811 // Check if the DefMI meet the requirement
4812 // described in the III. If yes, set the ImmMO and RegMO accordingly.
4813 MachineOperand *ImmMO = nullptr;
4814 MachineOperand *RegMO = nullptr;
4815 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4816 return false;
4817 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4818
4819 // As we get the Imm operand now, we need to check if the ImmMO meet
4820 // the requirement described in the III. If yes set the Imm.
4821 int64_t Imm = 0;
4822 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4823 return false;
4824
4825 bool IsFwdFeederRegKilled = false;
4826 bool SeenIntermediateUse = false;
4827 // Check if the RegMO can be forwarded to MI.
4828 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4829 IsFwdFeederRegKilled, SeenIntermediateUse))
4830 return false;
4831
4832 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4833 bool PostRA = !MRI.isSSA();
4834
4835 // We know that, the MI and DefMI both meet the pattern, and
4836 // the Imm also meet the requirement with the new Imm-form.
4837 // It is safe to do the transformation now.
4838 LLVM_DEBUG(dbgs() << "Replacing indexed instruction:\n");
4839 LLVM_DEBUG(MI.dump());
4840 LLVM_DEBUG(dbgs() << "Fed by:\n");
4841 LLVM_DEBUG(DefMI.dump());
4842
4843 // Update the base reg first.
4844 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4845 false, false,
4846 RegMO->isKill());
4847
4848 // Then, update the imm.
4849 if (ImmMO->isImm()) {
4850 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4851 // directly.
4852 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4853 }
4854 else {
4855 // Otherwise, it is Constant Pool Index(CPI) or Global,
4856 // which is relocation in fact. We need to replace the special zero
4857 // register with ImmMO.
4858 // Before that, we need to fixup the target flags for imm.
4859 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4860 if (DefMI.getOpcode() == PPC::ADDItocL)
4861 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4862
4863 // MI didn't have the interface such as MI.setOperand(i) though
4864 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4865 // ImmMO, we need to remove ZERO operand and all the operands behind it,
4866 // and, add the ImmMO, then, move back all the operands behind ZERO.
4867 SmallVector<MachineOperand, 2> MOps;
4868 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4869 MOps.push_back(MI.getOperand(i));
4870 MI.removeOperand(i);
4871 }
4872
4873 // Remove the last MO in the list, which is ZERO operand in fact.
4874 MOps.pop_back();
4875 // Add the imm operand.
4876 MI.addOperand(*ImmMO);
4877 // Now add the rest back.
4878 for (auto &MO : MOps)
4879 MI.addOperand(MO);
4880 }
4881
4882 // Update the opcode.
4883 MI.setDesc(get(III.ImmOpcode));
4884
4885 if (PostRA)
4886 recomputeLivenessFlags(*MI.getParent());
4887 LLVM_DEBUG(dbgs() << "With:\n");
4888 LLVM_DEBUG(MI.dump());
4889
4890 return true;
4891 }
4892
transformToImmFormFedByLI(MachineInstr & MI,const ImmInstrInfo & III,unsigned ConstantOpNo,MachineInstr & DefMI) const4893 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4894 const ImmInstrInfo &III,
4895 unsigned ConstantOpNo,
4896 MachineInstr &DefMI) const {
4897 // DefMI must be LI or LI8.
4898 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4899 !DefMI.getOperand(1).isImm())
4900 return false;
4901
4902 // Get Imm operand and Sign-extend to 64-bits.
4903 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4904
4905 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4906 bool PostRA = !MRI.isSSA();
4907 // Exit early if we can't convert this.
4908 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4909 return false;
4910 if (Imm % III.ImmMustBeMultipleOf)
4911 return false;
4912 if (III.TruncateImmTo)
4913 Imm &= ((1 << III.TruncateImmTo) - 1);
4914 if (III.SignedImm) {
4915 APInt ActualValue(64, Imm, true);
4916 if (!ActualValue.isSignedIntN(III.ImmWidth))
4917 return false;
4918 } else {
4919 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4920 if ((uint64_t)Imm > UnsignedMax)
4921 return false;
4922 }
4923
4924 // If we're post-RA, the instructions don't agree on whether register zero is
4925 // special, we can transform this as long as the register operand that will
4926 // end up in the location where zero is special isn't R0.
4927 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4928 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4929 III.ZeroIsSpecialNew + 1;
4930 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4931 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4932 // If R0 is in the operand where zero is special for the new instruction,
4933 // it is unsafe to transform if the constant operand isn't that operand.
4934 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4935 ConstantOpNo != III.ZeroIsSpecialNew)
4936 return false;
4937 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4938 ConstantOpNo != PosForOrigZero)
4939 return false;
4940 }
4941
4942 unsigned Opc = MI.getOpcode();
4943 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4944 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4945 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4946 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4947 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4948 Opc == PPC::SRD || Opc == PPC::SRD_rec;
4949 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4950 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4951 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4952 Opc == PPC::SRD_rec;
4953
4954 LLVM_DEBUG(dbgs() << "Replacing reg+reg instruction: ");
4955 LLVM_DEBUG(MI.dump());
4956 LLVM_DEBUG(dbgs() << "Fed by load-immediate: ");
4957 LLVM_DEBUG(DefMI.dump());
4958 MI.setDesc(get(III.ImmOpcode));
4959 if (ConstantOpNo == III.OpNoForForwarding) {
4960 // Converting shifts to immediate form is a bit tricky since they may do
4961 // one of three things:
4962 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4963 // 2. If the shift amount is zero, the result is unchanged (save for maybe
4964 // setting CR0)
4965 // 3. If the shift amount is in [1, OpSize), it's just a shift
4966 if (SpecialShift32 || SpecialShift64) {
4967 LoadImmediateInfo LII;
4968 LII.Imm = 0;
4969 LII.SetCR = SetCR;
4970 LII.Is64Bit = SpecialShift64;
4971 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
4972 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
4973 replaceInstrWithLI(MI, LII);
4974 // Shifts by zero don't change the value. If we don't need to set CR0,
4975 // just convert this to a COPY. Can't do this post-RA since we've already
4976 // cleaned up the copies.
4977 else if (!SetCR && ShAmt == 0 && !PostRA) {
4978 MI.removeOperand(2);
4979 MI.setDesc(get(PPC::COPY));
4980 } else {
4981 // The 32 bit and 64 bit instructions are quite different.
4982 if (SpecialShift32) {
4983 // Left shifts use (N, 0, 31-N).
4984 // Right shifts use (32-N, N, 31) if 0 < N < 32.
4985 // use (0, 0, 31) if N == 0.
4986 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
4987 uint64_t MB = RightShift ? ShAmt : 0;
4988 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
4989 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4990 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
4991 .addImm(ME);
4992 } else {
4993 // Left shifts use (N, 63-N).
4994 // Right shifts use (64-N, N) if 0 < N < 64.
4995 // use (0, 0) if N == 0.
4996 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
4997 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
4998 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4999 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
5000 }
5001 }
5002 } else
5003 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5004 }
5005 // Convert commutative instructions (switch the operands and convert the
5006 // desired one to an immediate.
5007 else if (III.IsCommutative) {
5008 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5009 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
5010 } else
5011 llvm_unreachable("Should have exited early!");
5012
5013 // For instructions for which the constant register replaces a different
5014 // operand than where the immediate goes, we need to swap them.
5015 if (III.OpNoForForwarding != III.ImmOpNo)
5016 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
5017
5018 // If the special R0/X0 register index are different for original instruction
5019 // and new instruction, we need to fix up the register class in new
5020 // instruction.
5021 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5022 if (III.ZeroIsSpecialNew) {
5023 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
5024 // need to fix up register class.
5025 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
5026 if (RegToModify.isVirtual()) {
5027 const TargetRegisterClass *NewRC =
5028 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
5029 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
5030 MRI.setRegClass(RegToModify, NewRC);
5031 }
5032 }
5033 }
5034
5035 if (PostRA)
5036 recomputeLivenessFlags(*MI.getParent());
5037
5038 LLVM_DEBUG(dbgs() << "With: ");
5039 LLVM_DEBUG(MI.dump());
5040 LLVM_DEBUG(dbgs() << "\n");
5041 return true;
5042 }
5043
5044 const TargetRegisterClass *
updatedRC(const TargetRegisterClass * RC) const5045 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
5046 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
5047 return &PPC::VSRCRegClass;
5048 return RC;
5049 }
5050
getRecordFormOpcode(unsigned Opcode)5051 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
5052 return PPC::getRecordFormOpcode(Opcode);
5053 }
5054
isOpZeroOfSubwordPreincLoad(int Opcode)5055 static bool isOpZeroOfSubwordPreincLoad(int Opcode) {
5056 return (Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 ||
5057 Opcode == PPC::LBZUX8 || Opcode == PPC::LHZU ||
5058 Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 ||
5059 Opcode == PPC::LHZUX8);
5060 }
5061
5062 // This function checks for sign extension from 32 bits to 64 bits.
definedBySignExtendingOp(const unsigned Reg,const MachineRegisterInfo * MRI)5063 static bool definedBySignExtendingOp(const unsigned Reg,
5064 const MachineRegisterInfo *MRI) {
5065 if (!Register::isVirtualRegister(Reg))
5066 return false;
5067
5068 MachineInstr *MI = MRI->getVRegDef(Reg);
5069 if (!MI)
5070 return false;
5071
5072 int Opcode = MI->getOpcode();
5073 const PPCInstrInfo *TII =
5074 MI->getMF()->getSubtarget<PPCSubtarget>().getInstrInfo();
5075 if (TII->isSExt32To64(Opcode))
5076 return true;
5077
5078 // The first def of LBZU/LHZU is sign extended.
5079 if (isOpZeroOfSubwordPreincLoad(Opcode) && MI->getOperand(0).getReg() == Reg)
5080 return true;
5081
5082 // RLDICL generates sign-extended output if it clears at least
5083 // 33 bits from the left (MSB).
5084 if (Opcode == PPC::RLDICL && MI->getOperand(3).getImm() >= 33)
5085 return true;
5086
5087 // If at least one bit from left in a lower word is masked out,
5088 // all of 0 to 32-th bits of the output are cleared.
5089 // Hence the output is already sign extended.
5090 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5091 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
5092 MI->getOperand(3).getImm() > 0 &&
5093 MI->getOperand(3).getImm() <= MI->getOperand(4).getImm())
5094 return true;
5095
5096 // If the most significant bit of immediate in ANDIS is zero,
5097 // all of 0 to 32-th bits are cleared.
5098 if (Opcode == PPC::ANDIS_rec || Opcode == PPC::ANDIS8_rec) {
5099 uint16_t Imm = MI->getOperand(2).getImm();
5100 if ((Imm & 0x8000) == 0)
5101 return true;
5102 }
5103
5104 return false;
5105 }
5106
5107 // This function checks the machine instruction that defines the input register
5108 // Reg. If that machine instruction always outputs a value that has only zeros
5109 // in the higher 32 bits then this function will return true.
definedByZeroExtendingOp(const unsigned Reg,const MachineRegisterInfo * MRI)5110 static bool definedByZeroExtendingOp(const unsigned Reg,
5111 const MachineRegisterInfo *MRI) {
5112 if (!Register::isVirtualRegister(Reg))
5113 return false;
5114
5115 MachineInstr *MI = MRI->getVRegDef(Reg);
5116 if (!MI)
5117 return false;
5118
5119 int Opcode = MI->getOpcode();
5120 const PPCInstrInfo *TII =
5121 MI->getMF()->getSubtarget<PPCSubtarget>().getInstrInfo();
5122 if (TII->isZExt32To64(Opcode))
5123 return true;
5124
5125 // The first def of LBZU/LHZU/LWZU are zero extended.
5126 if ((isOpZeroOfSubwordPreincLoad(Opcode) || Opcode == PPC::LWZU ||
5127 Opcode == PPC::LWZUX || Opcode == PPC::LWZU8 || Opcode == PPC::LWZUX8) &&
5128 MI->getOperand(0).getReg() == Reg)
5129 return true;
5130
5131 // The 16-bit immediate is sign-extended in li/lis.
5132 // If the most significant bit is zero, all higher bits are zero.
5133 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
5134 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
5135 int64_t Imm = MI->getOperand(1).getImm();
5136 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
5137 return true;
5138 }
5139
5140 // We have some variations of rotate-and-mask instructions
5141 // that clear higher 32-bits.
5142 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
5143 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
5144 Opcode == PPC::RLDICL_32_64) &&
5145 MI->getOperand(3).getImm() >= 32)
5146 return true;
5147
5148 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
5149 MI->getOperand(3).getImm() >= 32 &&
5150 MI->getOperand(3).getImm() <= 63 - MI->getOperand(2).getImm())
5151 return true;
5152
5153 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5154 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
5155 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
5156 MI->getOperand(3).getImm() <= MI->getOperand(4).getImm())
5157 return true;
5158
5159 return false;
5160 }
5161
5162 // This function returns true if the input MachineInstr is a TOC save
5163 // instruction.
isTOCSaveMI(const MachineInstr & MI) const5164 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
5165 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
5166 return false;
5167 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5168 unsigned StackOffset = MI.getOperand(1).getImm();
5169 Register StackReg = MI.getOperand(2).getReg();
5170 Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
5171 if (StackReg == SPReg && StackOffset == TOCSaveOffset)
5172 return true;
5173
5174 return false;
5175 }
5176
5177 // We limit the max depth to track incoming values of PHIs or binary ops
5178 // (e.g. AND) to avoid excessive cost.
5179 const unsigned MAX_BINOP_DEPTH = 1;
5180 // The isSignOrZeroExtended function is recursive. The parameter BinOpDepth
5181 // does not count all of the recursions. The parameter BinOpDepth is incremented
5182 // only when isSignOrZeroExtended calls itself more than once. This is done to
5183 // prevent expontential recursion. There is no parameter to track linear
5184 // recursion.
5185 std::pair<bool, bool>
isSignOrZeroExtended(const unsigned Reg,const unsigned BinOpDepth,const MachineRegisterInfo * MRI) const5186 PPCInstrInfo::isSignOrZeroExtended(const unsigned Reg,
5187 const unsigned BinOpDepth,
5188 const MachineRegisterInfo *MRI) const {
5189 if (!Register::isVirtualRegister(Reg))
5190 return std::pair<bool, bool>(false, false);
5191
5192 MachineInstr *MI = MRI->getVRegDef(Reg);
5193 if (!MI)
5194 return std::pair<bool, bool>(false, false);
5195
5196 bool IsSExt = definedBySignExtendingOp(Reg, MRI);
5197 bool IsZExt = definedByZeroExtendingOp(Reg, MRI);
5198
5199 // If we know the instruction always returns sign- and zero-extended result,
5200 // return here.
5201 if (IsSExt && IsZExt)
5202 return std::pair<bool, bool>(IsSExt, IsZExt);
5203
5204 switch (MI->getOpcode()) {
5205 case PPC::COPY: {
5206 Register SrcReg = MI->getOperand(1).getReg();
5207
5208 // In both ELFv1 and v2 ABI, method parameters and the return value
5209 // are sign- or zero-extended.
5210 const MachineFunction *MF = MI->getMF();
5211
5212 if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5213 // If this is a copy from another register, we recursively check source.
5214 auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5215 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5216 SrcExt.second || IsZExt);
5217 }
5218
5219 // From here on everything is SVR4ABI
5220 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
5221 // We check the ZExt/SExt flags for a method parameter.
5222 if (MI->getParent()->getBasicBlock() ==
5223 &MF->getFunction().getEntryBlock()) {
5224 Register VReg = MI->getOperand(0).getReg();
5225 if (MF->getRegInfo().isLiveIn(VReg)) {
5226 IsSExt |= FuncInfo->isLiveInSExt(VReg);
5227 IsZExt |= FuncInfo->isLiveInZExt(VReg);
5228 return std::pair<bool, bool>(IsSExt, IsZExt);
5229 }
5230 }
5231
5232 if (SrcReg != PPC::X3) {
5233 // If this is a copy from another register, we recursively check source.
5234 auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5235 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5236 SrcExt.second || IsZExt);
5237 }
5238
5239 // For a method return value, we check the ZExt/SExt flags in attribute.
5240 // We assume the following code sequence for method call.
5241 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
5242 // BL8_NOP @func,...
5243 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
5244 // %5 = COPY %x3; G8RC:%5
5245 const MachineBasicBlock *MBB = MI->getParent();
5246 std::pair<bool, bool> IsExtendPair = std::pair<bool, bool>(IsSExt, IsZExt);
5247 MachineBasicBlock::const_instr_iterator II =
5248 MachineBasicBlock::const_instr_iterator(MI);
5249 if (II == MBB->instr_begin() || (--II)->getOpcode() != PPC::ADJCALLSTACKUP)
5250 return IsExtendPair;
5251
5252 const MachineInstr &CallMI = *(--II);
5253 if (!CallMI.isCall() || !CallMI.getOperand(0).isGlobal())
5254 return IsExtendPair;
5255
5256 const Function *CalleeFn =
5257 dyn_cast_if_present<Function>(CallMI.getOperand(0).getGlobal());
5258 if (!CalleeFn)
5259 return IsExtendPair;
5260 const IntegerType *IntTy = dyn_cast<IntegerType>(CalleeFn->getReturnType());
5261 if (IntTy && IntTy->getBitWidth() <= 32) {
5262 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
5263 IsSExt |= Attrs.hasAttribute(Attribute::SExt);
5264 IsZExt |= Attrs.hasAttribute(Attribute::ZExt);
5265 return std::pair<bool, bool>(IsSExt, IsZExt);
5266 }
5267
5268 return IsExtendPair;
5269 }
5270
5271 // OR, XOR with 16-bit immediate does not change the upper 48 bits.
5272 // So, we track the operand register as we do for register copy.
5273 case PPC::ORI:
5274 case PPC::XORI:
5275 case PPC::ORI8:
5276 case PPC::XORI8: {
5277 Register SrcReg = MI->getOperand(1).getReg();
5278 auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5279 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5280 SrcExt.second || IsZExt);
5281 }
5282
5283 // OR, XOR with shifted 16-bit immediate does not change the upper
5284 // 32 bits. So, we track the operand register for zero extension.
5285 // For sign extension when the MSB of the immediate is zero, we also
5286 // track the operand register since the upper 33 bits are unchanged.
5287 case PPC::ORIS:
5288 case PPC::XORIS:
5289 case PPC::ORIS8:
5290 case PPC::XORIS8: {
5291 Register SrcReg = MI->getOperand(1).getReg();
5292 auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5293 uint16_t Imm = MI->getOperand(2).getImm();
5294 if (Imm & 0x8000)
5295 return std::pair<bool, bool>(false, SrcExt.second || IsZExt);
5296 else
5297 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5298 SrcExt.second || IsZExt);
5299 }
5300
5301 // If all incoming values are sign-/zero-extended,
5302 // the output of OR, ISEL or PHI is also sign-/zero-extended.
5303 case PPC::OR:
5304 case PPC::OR8:
5305 case PPC::ISEL:
5306 case PPC::PHI: {
5307 if (BinOpDepth >= MAX_BINOP_DEPTH)
5308 return std::pair<bool, bool>(false, false);
5309
5310 // The input registers for PHI are operand 1, 3, ...
5311 // The input registers for others are operand 1 and 2.
5312 unsigned OperandEnd = 3, OperandStride = 1;
5313 if (MI->getOpcode() == PPC::PHI) {
5314 OperandEnd = MI->getNumOperands();
5315 OperandStride = 2;
5316 }
5317
5318 IsSExt = true;
5319 IsZExt = true;
5320 for (unsigned I = 1; I != OperandEnd; I += OperandStride) {
5321 if (!MI->getOperand(I).isReg())
5322 return std::pair<bool, bool>(false, false);
5323
5324 Register SrcReg = MI->getOperand(I).getReg();
5325 auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth + 1, MRI);
5326 IsSExt &= SrcExt.first;
5327 IsZExt &= SrcExt.second;
5328 }
5329 return std::pair<bool, bool>(IsSExt, IsZExt);
5330 }
5331
5332 // If at least one of the incoming values of an AND is zero extended
5333 // then the output is also zero-extended. If both of the incoming values
5334 // are sign-extended then the output is also sign extended.
5335 case PPC::AND:
5336 case PPC::AND8: {
5337 if (BinOpDepth >= MAX_BINOP_DEPTH)
5338 return std::pair<bool, bool>(false, false);
5339
5340 Register SrcReg1 = MI->getOperand(1).getReg();
5341 Register SrcReg2 = MI->getOperand(2).getReg();
5342 auto Src1Ext = isSignOrZeroExtended(SrcReg1, BinOpDepth + 1, MRI);
5343 auto Src2Ext = isSignOrZeroExtended(SrcReg2, BinOpDepth + 1, MRI);
5344 return std::pair<bool, bool>(Src1Ext.first && Src2Ext.first,
5345 Src1Ext.second || Src2Ext.second);
5346 }
5347
5348 default:
5349 break;
5350 }
5351 return std::pair<bool, bool>(IsSExt, IsZExt);
5352 }
5353
isBDNZ(unsigned Opcode) const5354 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
5355 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
5356 }
5357
5358 namespace {
5359 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5360 MachineInstr *Loop, *EndLoop, *LoopCount;
5361 MachineFunction *MF;
5362 const TargetInstrInfo *TII;
5363 int64_t TripCount;
5364
5365 public:
PPCPipelinerLoopInfo(MachineInstr * Loop,MachineInstr * EndLoop,MachineInstr * LoopCount)5366 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
5367 MachineInstr *LoopCount)
5368 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
5369 MF(Loop->getParent()->getParent()),
5370 TII(MF->getSubtarget().getInstrInfo()) {
5371 // Inspect the Loop instruction up-front, as it may be deleted when we call
5372 // createTripCountGreaterCondition.
5373 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
5374 TripCount = LoopCount->getOperand(1).getImm();
5375 else
5376 TripCount = -1;
5377 }
5378
shouldIgnoreForPipelining(const MachineInstr * MI) const5379 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5380 // Only ignore the terminator.
5381 return MI == EndLoop;
5382 }
5383
createTripCountGreaterCondition(int TC,MachineBasicBlock & MBB,SmallVectorImpl<MachineOperand> & Cond)5384 std::optional<bool> createTripCountGreaterCondition(
5385 int TC, MachineBasicBlock &MBB,
5386 SmallVectorImpl<MachineOperand> &Cond) override {
5387 if (TripCount == -1) {
5388 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5389 // so we don't need to generate any thing here.
5390 Cond.push_back(MachineOperand::CreateImm(0));
5391 Cond.push_back(MachineOperand::CreateReg(
5392 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
5393 true));
5394 return {};
5395 }
5396
5397 return TripCount > TC;
5398 }
5399
setPreheader(MachineBasicBlock * NewPreheader)5400 void setPreheader(MachineBasicBlock *NewPreheader) override {
5401 // Do nothing. We want the LOOP setup instruction to stay in the *old*
5402 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
5403 }
5404
adjustTripCount(int TripCountAdjust)5405 void adjustTripCount(int TripCountAdjust) override {
5406 // If the loop trip count is a compile-time value, then just change the
5407 // value.
5408 if (LoopCount->getOpcode() == PPC::LI8 ||
5409 LoopCount->getOpcode() == PPC::LI) {
5410 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
5411 LoopCount->getOperand(1).setImm(TripCount);
5412 return;
5413 }
5414
5415 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5416 // so we don't need to generate any thing here.
5417 }
5418
disposed()5419 void disposed() override {
5420 Loop->eraseFromParent();
5421 // Ensure the loop setup instruction is deleted too.
5422 LoopCount->eraseFromParent();
5423 }
5424 };
5425 } // namespace
5426
5427 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
analyzeLoopForPipelining(MachineBasicBlock * LoopBB) const5428 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5429 // We really "analyze" only hardware loops right now.
5430 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
5431 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
5432 if (Preheader == LoopBB)
5433 Preheader = *std::next(LoopBB->pred_begin());
5434 MachineFunction *MF = Preheader->getParent();
5435
5436 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
5437 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5438 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
5439 Register LoopCountReg = LoopInst->getOperand(0).getReg();
5440 MachineRegisterInfo &MRI = MF->getRegInfo();
5441 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
5442 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
5443 }
5444 }
5445 return nullptr;
5446 }
5447
findLoopInstr(MachineBasicBlock & PreHeader,SmallPtrSet<MachineBasicBlock *,8> & Visited) const5448 MachineInstr *PPCInstrInfo::findLoopInstr(
5449 MachineBasicBlock &PreHeader,
5450 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
5451
5452 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
5453
5454 // The loop set-up instruction should be in preheader
5455 for (auto &I : PreHeader.instrs())
5456 if (I.getOpcode() == LOOPi)
5457 return &I;
5458 return nullptr;
5459 }
5460
5461 // Return true if get the base operand, byte offset of an instruction and the
5462 // memory width. Width is the size of memory that is being loaded/stored.
getMemOperandWithOffsetWidth(const MachineInstr & LdSt,const MachineOperand * & BaseReg,int64_t & Offset,unsigned & Width,const TargetRegisterInfo * TRI) const5463 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
5464 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
5465 unsigned &Width, const TargetRegisterInfo *TRI) const {
5466 if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
5467 return false;
5468
5469 // Handle only loads/stores with base register followed by immediate offset.
5470 if (!LdSt.getOperand(1).isImm() ||
5471 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5472 return false;
5473 if (!LdSt.getOperand(1).isImm() ||
5474 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5475 return false;
5476
5477 if (!LdSt.hasOneMemOperand())
5478 return false;
5479
5480 Width = (*LdSt.memoperands_begin())->getSize();
5481 Offset = LdSt.getOperand(1).getImm();
5482 BaseReg = &LdSt.getOperand(2);
5483 return true;
5484 }
5485
areMemAccessesTriviallyDisjoint(const MachineInstr & MIa,const MachineInstr & MIb) const5486 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
5487 const MachineInstr &MIa, const MachineInstr &MIb) const {
5488 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
5489 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
5490
5491 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
5492 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
5493 return false;
5494
5495 // Retrieve the base register, offset from the base register and width. Width
5496 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
5497 // base registers are identical, and the offset of a lower memory access +
5498 // the width doesn't overlap the offset of a higher memory access,
5499 // then the memory accesses are different.
5500 const TargetRegisterInfo *TRI = &getRegisterInfo();
5501 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
5502 int64_t OffsetA = 0, OffsetB = 0;
5503 unsigned int WidthA = 0, WidthB = 0;
5504 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
5505 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
5506 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
5507 int LowOffset = std::min(OffsetA, OffsetB);
5508 int HighOffset = std::max(OffsetA, OffsetB);
5509 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
5510 if (LowOffset + LowWidth <= HighOffset)
5511 return true;
5512 }
5513 }
5514 return false;
5515 }
5516