1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineMemOperand.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/PseudoSourceValue.h"
30 #include "llvm/CodeGen/ScheduleDAG.h"
31 #include "llvm/CodeGen/SlotIndexes.h"
32 #include "llvm/CodeGen/StackMaps.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCInst.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
40
41 using namespace llvm;
42
43 #define DEBUG_TYPE "ppc-instr-info"
44
45 #define GET_INSTRMAP_INFO
46 #define GET_INSTRINFO_CTOR_DTOR
47 #include "PPCGenInstrInfo.inc"
48
49 STATISTIC(NumStoreSPILLVSRRCAsVec,
50 "Number of spillvsrrc spilled to stack as vec");
51 STATISTIC(NumStoreSPILLVSRRCAsGpr,
52 "Number of spillvsrrc spilled to stack as gpr");
53 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
54 STATISTIC(CmpIselsConverted,
55 "Number of ISELs that depend on comparison of constants converted");
56 STATISTIC(MissedConvertibleImmediateInstrs,
57 "Number of compare-immediate instructions fed by constants");
58 STATISTIC(NumRcRotatesConvertedToRcAnd,
59 "Number of record-form rotates converted to record-form andi");
60
61 static cl::
62 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
63 cl::desc("Disable analysis for CTR loops"));
64
65 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
66 cl::desc("Disable compare instruction optimization"), cl::Hidden);
67
68 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
69 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
70 cl::Hidden);
71
72 static cl::opt<bool>
73 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
74 cl::desc("Use the old (incorrect) instruction latency calculation"));
75
76 // Pin the vtable to this file.
anchor()77 void PPCInstrInfo::anchor() {}
78
PPCInstrInfo(PPCSubtarget & STI)79 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
80 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
81 /* CatchRetOpcode */ -1,
82 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
83 Subtarget(STI), RI(STI.getTargetMachine()) {}
84
85 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
86 /// this target when scheduling the DAG.
87 ScheduleHazardRecognizer *
CreateTargetHazardRecognizer(const TargetSubtargetInfo * STI,const ScheduleDAG * DAG) const88 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
89 const ScheduleDAG *DAG) const {
90 unsigned Directive =
91 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
92 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
93 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
94 const InstrItineraryData *II =
95 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
96 return new ScoreboardHazardRecognizer(II, DAG);
97 }
98
99 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
100 }
101
102 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
103 /// to use for this target when scheduling the DAG.
104 ScheduleHazardRecognizer *
CreateTargetPostRAHazardRecognizer(const InstrItineraryData * II,const ScheduleDAG * DAG) const105 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
106 const ScheduleDAG *DAG) const {
107 unsigned Directive =
108 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
109
110 // FIXME: Leaving this as-is until we have POWER9 scheduling info
111 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
112 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
113
114 // Most subtargets use a PPC970 recognizer.
115 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
116 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
117 assert(DAG->TII && "No InstrInfo?");
118
119 return new PPCHazardRecognizer970(*DAG);
120 }
121
122 return new ScoreboardHazardRecognizer(II, DAG);
123 }
124
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr & MI,unsigned * PredCost) const125 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
126 const MachineInstr &MI,
127 unsigned *PredCost) const {
128 if (!ItinData || UseOldLatencyCalc)
129 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
130
131 // The default implementation of getInstrLatency calls getStageLatency, but
132 // getStageLatency does not do the right thing for us. While we have
133 // itinerary, most cores are fully pipelined, and so the itineraries only
134 // express the first part of the pipeline, not every stage. Instead, we need
135 // to use the listed output operand cycle number (using operand 0 here, which
136 // is an output).
137
138 unsigned Latency = 1;
139 unsigned DefClass = MI.getDesc().getSchedClass();
140 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
141 const MachineOperand &MO = MI.getOperand(i);
142 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
143 continue;
144
145 int Cycle = ItinData->getOperandCycle(DefClass, i);
146 if (Cycle < 0)
147 continue;
148
149 Latency = std::max(Latency, (unsigned) Cycle);
150 }
151
152 return Latency;
153 }
154
getOperandLatency(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const155 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
156 const MachineInstr &DefMI, unsigned DefIdx,
157 const MachineInstr &UseMI,
158 unsigned UseIdx) const {
159 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
160 UseMI, UseIdx);
161
162 if (!DefMI.getParent())
163 return Latency;
164
165 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
166 Register Reg = DefMO.getReg();
167
168 bool IsRegCR;
169 if (Register::isVirtualRegister(Reg)) {
170 const MachineRegisterInfo *MRI =
171 &DefMI.getParent()->getParent()->getRegInfo();
172 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
173 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
174 } else {
175 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
176 PPC::CRBITRCRegClass.contains(Reg);
177 }
178
179 if (UseMI.isBranch() && IsRegCR) {
180 if (Latency < 0)
181 Latency = getInstrLatency(ItinData, DefMI);
182
183 // On some cores, there is an additional delay between writing to a condition
184 // register, and using it from a branch.
185 unsigned Directive = Subtarget.getCPUDirective();
186 switch (Directive) {
187 default: break;
188 case PPC::DIR_7400:
189 case PPC::DIR_750:
190 case PPC::DIR_970:
191 case PPC::DIR_E5500:
192 case PPC::DIR_PWR4:
193 case PPC::DIR_PWR5:
194 case PPC::DIR_PWR5X:
195 case PPC::DIR_PWR6:
196 case PPC::DIR_PWR6X:
197 case PPC::DIR_PWR7:
198 case PPC::DIR_PWR8:
199 // FIXME: Is this needed for POWER9?
200 Latency += 2;
201 break;
202 }
203 }
204
205 return Latency;
206 }
207
208 /// This is an architecture-specific helper function of reassociateOps.
209 /// Set special operand attributes for new instructions after reassociation.
setSpecialOperandAttr(MachineInstr & OldMI1,MachineInstr & OldMI2,MachineInstr & NewMI1,MachineInstr & NewMI2) const210 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
211 MachineInstr &OldMI2,
212 MachineInstr &NewMI1,
213 MachineInstr &NewMI2) const {
214 // Propagate FP flags from the original instructions.
215 // But clear poison-generating flags because those may not be valid now.
216 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
217 NewMI1.setFlags(IntersectedFlags);
218 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
219 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
220 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
221
222 NewMI2.setFlags(IntersectedFlags);
223 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
224 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
225 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
226 }
227
setSpecialOperandAttr(MachineInstr & MI,uint16_t Flags) const228 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
229 uint16_t Flags) const {
230 MI.setFlags(Flags);
231 MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
232 MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
233 MI.clearFlag(MachineInstr::MIFlag::IsExact);
234 }
235
236 // This function does not list all associative and commutative operations, but
237 // only those worth feeding through the machine combiner in an attempt to
238 // reduce the critical path. Mostly, this means floating-point operations,
239 // because they have high latencies(>=5) (compared to other operations, such as
240 // and/or, which are also associative and commutative, but have low latencies).
isAssociativeAndCommutative(const MachineInstr & Inst) const241 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
242 switch (Inst.getOpcode()) {
243 // Floating point:
244 // FP Add:
245 case PPC::FADD:
246 case PPC::FADDS:
247 // FP Multiply:
248 case PPC::FMUL:
249 case PPC::FMULS:
250 // Altivec Add:
251 case PPC::VADDFP:
252 // VSX Add:
253 case PPC::XSADDDP:
254 case PPC::XVADDDP:
255 case PPC::XVADDSP:
256 case PPC::XSADDSP:
257 // VSX Multiply:
258 case PPC::XSMULDP:
259 case PPC::XVMULDP:
260 case PPC::XVMULSP:
261 case PPC::XSMULSP:
262 // QPX Add:
263 case PPC::QVFADD:
264 case PPC::QVFADDS:
265 case PPC::QVFADDSs:
266 // QPX Multiply:
267 case PPC::QVFMUL:
268 case PPC::QVFMULS:
269 case PPC::QVFMULSs:
270 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
271 Inst.getFlag(MachineInstr::MIFlag::FmNsz);
272 // Fixed point:
273 // Multiply:
274 case PPC::MULHD:
275 case PPC::MULLD:
276 case PPC::MULHW:
277 case PPC::MULLW:
278 return true;
279 default:
280 return false;
281 }
282 }
283
284 #define InfoArrayIdxFMAInst 0
285 #define InfoArrayIdxFAddInst 1
286 #define InfoArrayIdxFMULInst 2
287 #define InfoArrayIdxAddOpIdx 3
288 #define InfoArrayIdxMULOpIdx 4
289 // Array keeps info for FMA instructions:
290 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
291 // Index 1(InfoArrayIdxFAddInst): ADD instruction assoaicted with FMA;
292 // Index 2(InfoArrayIdxFMULInst): MUL instruction assoaicted with FMA;
293 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
294 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
295 // second MUL operand index is plus 1.
296 static const uint16_t FMAOpIdxInfo[][5] = {
297 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
298 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2},
299 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2},
300 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
301 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
302 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
303 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
304 {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
305 {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
306
307 // Check if an opcode is a FMA instruction. If it is, return the index in array
308 // FMAOpIdxInfo. Otherwise, return -1.
getFMAOpIdxInfo(unsigned Opcode) const309 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
310 for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++)
311 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
312 return I;
313 return -1;
314 }
315
316 // Try to reassociate FMA chains like below:
317 //
318 // Pattern 1:
319 // A = FADD X, Y (Leaf)
320 // B = FMA A, M21, M22 (Prev)
321 // C = FMA B, M31, M32 (Root)
322 // -->
323 // A = FMA X, M21, M22
324 // B = FMA Y, M31, M32
325 // C = FADD A, B
326 //
327 // Pattern 2:
328 // A = FMA X, M11, M12 (Leaf)
329 // B = FMA A, M21, M22 (Prev)
330 // C = FMA B, M31, M32 (Root)
331 // -->
332 // A = FMUL M11, M12
333 // B = FMA X, M21, M22
334 // D = FMA A, M31, M32
335 // C = FADD B, D
336 //
337 // breaking the dependency between A and B, allowing FMA to be executed in
338 // parallel (or back-to-back in a pipeline) instead of depending on each other.
getFMAPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns) const339 bool PPCInstrInfo::getFMAPatterns(
340 MachineInstr &Root,
341 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
342 MachineBasicBlock *MBB = Root.getParent();
343 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
344
345 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
346 for (const auto &MO : Instr.explicit_operands())
347 if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
348 return false;
349 return true;
350 };
351
352 auto IsReassociable = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
353 bool IsLeaf, bool IsAdd) {
354 int16_t Idx = -1;
355 if (!IsAdd) {
356 Idx = getFMAOpIdxInfo(Instr.getOpcode());
357 if (Idx < 0)
358 return false;
359 } else if (Instr.getOpcode() !=
360 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())]
361 [InfoArrayIdxFAddInst])
362 return false;
363
364 // Instruction can be reassociated.
365 // fast math flags may prohibit reassociation.
366 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
367 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
368 return false;
369
370 // Instruction operands are virtual registers for reassociation.
371 if (!IsAllOpsVirtualReg(Instr))
372 return false;
373
374 if (IsAdd && IsLeaf)
375 return true;
376
377 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
378
379 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
380 MachineInstr *MIAdd = MRI.getUniqueVRegDef(OpAdd.getReg());
381 // If 'add' operand's def is not in current block, don't do ILP related opt.
382 if (!MIAdd || MIAdd->getParent() != MBB)
383 return false;
384
385 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
386 // as this fma will be changed later.
387 return IsLeaf ? true : MRI.hasOneNonDBGUse(OpAdd.getReg());
388 };
389
390 int16_t AddOpIdx = -1;
391 // Root must be a valid FMA like instruction.
392 if (!IsReassociable(Root, AddOpIdx, false, false))
393 return false;
394
395 assert((AddOpIdx >= 0) && "add operand index not right!");
396
397 Register RegB = Root.getOperand(AddOpIdx).getReg();
398 MachineInstr *Prev = MRI.getUniqueVRegDef(RegB);
399
400 // Prev must be a valid FMA like instruction.
401 AddOpIdx = -1;
402 if (!IsReassociable(*Prev, AddOpIdx, false, false))
403 return false;
404
405 assert((AddOpIdx >= 0) && "add operand index not right!");
406
407 Register RegA = Prev->getOperand(AddOpIdx).getReg();
408 MachineInstr *Leaf = MRI.getUniqueVRegDef(RegA);
409 AddOpIdx = -1;
410 if (IsReassociable(*Leaf, AddOpIdx, true, false)) {
411 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
412 return true;
413 }
414 if (IsReassociable(*Leaf, AddOpIdx, true, true)) {
415 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
416 return true;
417 }
418 return false;
419 }
420
getMachineCombinerPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns) const421 bool PPCInstrInfo::getMachineCombinerPatterns(
422 MachineInstr &Root,
423 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
424 // Using the machine combiner in this way is potentially expensive, so
425 // restrict to when aggressive optimizations are desired.
426 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
427 return false;
428
429 if (getFMAPatterns(Root, Patterns))
430 return true;
431
432 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
433 }
434
genAlternativeCodeSequence(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const435 void PPCInstrInfo::genAlternativeCodeSequence(
436 MachineInstr &Root, MachineCombinerPattern Pattern,
437 SmallVectorImpl<MachineInstr *> &InsInstrs,
438 SmallVectorImpl<MachineInstr *> &DelInstrs,
439 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
440 switch (Pattern) {
441 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
442 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
443 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
444 break;
445 default:
446 // Reassociate default patterns.
447 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
448 DelInstrs, InstrIdxForVirtReg);
449 break;
450 }
451 }
452
453 // Currently, only handle two patterns REASSOC_XY_AMM_BMM and
454 // REASSOC_XMM_AMM_BMM. See comments for getFMAPatterns.
reassociateFMA(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const455 void PPCInstrInfo::reassociateFMA(
456 MachineInstr &Root, MachineCombinerPattern Pattern,
457 SmallVectorImpl<MachineInstr *> &InsInstrs,
458 SmallVectorImpl<MachineInstr *> &DelInstrs,
459 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
460 MachineFunction *MF = Root.getMF();
461 MachineRegisterInfo &MRI = MF->getRegInfo();
462 MachineOperand &OpC = Root.getOperand(0);
463 Register RegC = OpC.getReg();
464 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
465 MRI.constrainRegClass(RegC, RC);
466
467 unsigned FmaOp = Root.getOpcode();
468 int16_t Idx = getFMAOpIdxInfo(FmaOp);
469 assert(Idx >= 0 && "Root must be a FMA instruction");
470
471 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
472 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
473 MachineInstr *Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
474 MachineInstr *Leaf =
475 MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
476 uint16_t IntersectedFlags =
477 Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
478
479 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
480 bool &KillFlag) {
481 Reg = Operand.getReg();
482 MRI.constrainRegClass(Reg, RC);
483 KillFlag = Operand.isKill();
484 };
485
486 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
487 Register &MulOp2, bool &MulOp1KillFlag,
488 bool &MulOp2KillFlag) {
489 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
490 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
491 };
492
493 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32;
494 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
495 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false;
496
497 GetFMAInstrInfo(Root, RegM31, RegM32, KillM31, KillM32);
498 GetFMAInstrInfo(*Prev, RegM21, RegM22, KillM21, KillM22);
499
500 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
501 GetFMAInstrInfo(*Leaf, RegM11, RegM12, KillM11, KillM12);
502 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
503 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
504 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
505 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
506 }
507
508 // Create new virtual registers for the new results instead of
509 // recycling legacy ones because the MachineCombiner's computation of the
510 // critical path requires a new register definition rather than an existing
511 // one.
512 Register NewVRA = MRI.createVirtualRegister(RC);
513 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
514
515 Register NewVRB = MRI.createVirtualRegister(RC);
516 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
517
518 Register NewVRD = 0;
519 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
520 NewVRD = MRI.createVirtualRegister(RC);
521 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
522 }
523
524 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
525 Register RegMul1, bool KillRegMul1,
526 Register RegMul2, bool KillRegMul2) {
527 MI->getOperand(AddOpIdx).setReg(RegAdd);
528 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
529 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
530 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
531 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
532 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
533 };
534
535 if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
536 // Create new instructions for insertion.
537 MachineInstrBuilder MINewB =
538 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
539 .addReg(RegX, getKillRegState(KillX))
540 .addReg(RegM21, getKillRegState(KillM21))
541 .addReg(RegM22, getKillRegState(KillM22));
542 MachineInstrBuilder MINewA =
543 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
544 .addReg(RegY, getKillRegState(KillY))
545 .addReg(RegM31, getKillRegState(KillM31))
546 .addReg(RegM32, getKillRegState(KillM32));
547 // If AddOpIdx is not 1, adjust the order.
548 if (AddOpIdx != 1) {
549 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
550 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
551 }
552
553 MachineInstrBuilder MINewC =
554 BuildMI(*MF, Root.getDebugLoc(),
555 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
556 .addReg(NewVRB, getKillRegState(true))
557 .addReg(NewVRA, getKillRegState(true));
558
559 // Update flags for newly created instructions.
560 setSpecialOperandAttr(*MINewA, IntersectedFlags);
561 setSpecialOperandAttr(*MINewB, IntersectedFlags);
562 setSpecialOperandAttr(*MINewC, IntersectedFlags);
563
564 // Record new instructions for insertion.
565 InsInstrs.push_back(MINewA);
566 InsInstrs.push_back(MINewB);
567 InsInstrs.push_back(MINewC);
568 } else if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
569 assert(NewVRD && "new FMA register not created!");
570 // Create new instructions for insertion.
571 MachineInstrBuilder MINewA =
572 BuildMI(*MF, Leaf->getDebugLoc(),
573 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
574 .addReg(RegM11, getKillRegState(KillM11))
575 .addReg(RegM12, getKillRegState(KillM12));
576 MachineInstrBuilder MINewB =
577 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
578 .addReg(RegX, getKillRegState(KillX))
579 .addReg(RegM21, getKillRegState(KillM21))
580 .addReg(RegM22, getKillRegState(KillM22));
581 MachineInstrBuilder MINewD =
582 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
583 .addReg(NewVRA, getKillRegState(true))
584 .addReg(RegM31, getKillRegState(KillM31))
585 .addReg(RegM32, getKillRegState(KillM32));
586 // If AddOpIdx is not 1, adjust the order.
587 if (AddOpIdx != 1) {
588 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
589 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
590 KillM32);
591 }
592
593 MachineInstrBuilder MINewC =
594 BuildMI(*MF, Root.getDebugLoc(),
595 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
596 .addReg(NewVRB, getKillRegState(true))
597 .addReg(NewVRD, getKillRegState(true));
598
599 // Update flags for newly created instructions.
600 setSpecialOperandAttr(*MINewA, IntersectedFlags);
601 setSpecialOperandAttr(*MINewB, IntersectedFlags);
602 setSpecialOperandAttr(*MINewD, IntersectedFlags);
603 setSpecialOperandAttr(*MINewC, IntersectedFlags);
604
605 // Record new instructions for insertion.
606 InsInstrs.push_back(MINewA);
607 InsInstrs.push_back(MINewB);
608 InsInstrs.push_back(MINewD);
609 InsInstrs.push_back(MINewC);
610 }
611
612 assert(!InsInstrs.empty() &&
613 "Insertion instructions set should not be empty!");
614
615 // Record old instructions for deletion.
616 DelInstrs.push_back(Leaf);
617 DelInstrs.push_back(Prev);
618 DelInstrs.push_back(&Root);
619 }
620
621 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
isCoalescableExtInstr(const MachineInstr & MI,Register & SrcReg,Register & DstReg,unsigned & SubIdx) const622 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
623 Register &SrcReg, Register &DstReg,
624 unsigned &SubIdx) const {
625 switch (MI.getOpcode()) {
626 default: return false;
627 case PPC::EXTSW:
628 case PPC::EXTSW_32:
629 case PPC::EXTSW_32_64:
630 SrcReg = MI.getOperand(1).getReg();
631 DstReg = MI.getOperand(0).getReg();
632 SubIdx = PPC::sub_32;
633 return true;
634 }
635 }
636
isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const637 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
638 int &FrameIndex) const {
639 unsigned Opcode = MI.getOpcode();
640 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
641 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
642
643 if (End != std::find(OpcodesForSpill, End, Opcode)) {
644 // Check for the operands added by addFrameReference (the immediate is the
645 // offset which defaults to 0).
646 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
647 MI.getOperand(2).isFI()) {
648 FrameIndex = MI.getOperand(2).getIndex();
649 return MI.getOperand(0).getReg();
650 }
651 }
652 return 0;
653 }
654
655 // For opcodes with the ReMaterializable flag set, this function is called to
656 // verify the instruction is really rematable.
isReallyTriviallyReMaterializable(const MachineInstr & MI,AliasAnalysis * AA) const657 bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
658 AliasAnalysis *AA) const {
659 switch (MI.getOpcode()) {
660 default:
661 // This function should only be called for opcodes with the ReMaterializable
662 // flag set.
663 llvm_unreachable("Unknown rematerializable operation!");
664 break;
665 case PPC::LI:
666 case PPC::LI8:
667 case PPC::LIS:
668 case PPC::LIS8:
669 case PPC::QVGPCI:
670 case PPC::ADDIStocHA:
671 case PPC::ADDIStocHA8:
672 case PPC::ADDItocL:
673 case PPC::LOAD_STACK_GUARD:
674 case PPC::XXLXORz:
675 case PPC::XXLXORspz:
676 case PPC::XXLXORdpz:
677 case PPC::XXLEQVOnes:
678 case PPC::V_SET0B:
679 case PPC::V_SET0H:
680 case PPC::V_SET0:
681 case PPC::V_SETALLONESB:
682 case PPC::V_SETALLONESH:
683 case PPC::V_SETALLONES:
684 case PPC::CRSET:
685 case PPC::CRUNSET:
686 return true;
687 }
688 return false;
689 }
690
isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const691 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
692 int &FrameIndex) const {
693 unsigned Opcode = MI.getOpcode();
694 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
695 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
696
697 if (End != std::find(OpcodesForSpill, End, Opcode)) {
698 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
699 MI.getOperand(2).isFI()) {
700 FrameIndex = MI.getOperand(2).getIndex();
701 return MI.getOperand(0).getReg();
702 }
703 }
704 return 0;
705 }
706
commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const707 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
708 unsigned OpIdx1,
709 unsigned OpIdx2) const {
710 MachineFunction &MF = *MI.getParent()->getParent();
711
712 // Normal instructions can be commuted the obvious way.
713 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
714 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
715 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
716 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
717 // changing the relative order of the mask operands might change what happens
718 // to the high-bits of the mask (and, thus, the result).
719
720 // Cannot commute if it has a non-zero rotate count.
721 if (MI.getOperand(3).getImm() != 0)
722 return nullptr;
723
724 // If we have a zero rotate count, we have:
725 // M = mask(MB,ME)
726 // Op0 = (Op1 & ~M) | (Op2 & M)
727 // Change this to:
728 // M = mask((ME+1)&31, (MB-1)&31)
729 // Op0 = (Op2 & ~M) | (Op1 & M)
730
731 // Swap op1/op2
732 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
733 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
734 Register Reg0 = MI.getOperand(0).getReg();
735 Register Reg1 = MI.getOperand(1).getReg();
736 Register Reg2 = MI.getOperand(2).getReg();
737 unsigned SubReg1 = MI.getOperand(1).getSubReg();
738 unsigned SubReg2 = MI.getOperand(2).getSubReg();
739 bool Reg1IsKill = MI.getOperand(1).isKill();
740 bool Reg2IsKill = MI.getOperand(2).isKill();
741 bool ChangeReg0 = false;
742 // If machine instrs are no longer in two-address forms, update
743 // destination register as well.
744 if (Reg0 == Reg1) {
745 // Must be two address instruction!
746 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
747 "Expecting a two-address instruction!");
748 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
749 Reg2IsKill = false;
750 ChangeReg0 = true;
751 }
752
753 // Masks.
754 unsigned MB = MI.getOperand(4).getImm();
755 unsigned ME = MI.getOperand(5).getImm();
756
757 // We can't commute a trivial mask (there is no way to represent an all-zero
758 // mask).
759 if (MB == 0 && ME == 31)
760 return nullptr;
761
762 if (NewMI) {
763 // Create a new instruction.
764 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
765 bool Reg0IsDead = MI.getOperand(0).isDead();
766 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
767 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
768 .addReg(Reg2, getKillRegState(Reg2IsKill))
769 .addReg(Reg1, getKillRegState(Reg1IsKill))
770 .addImm((ME + 1) & 31)
771 .addImm((MB - 1) & 31);
772 }
773
774 if (ChangeReg0) {
775 MI.getOperand(0).setReg(Reg2);
776 MI.getOperand(0).setSubReg(SubReg2);
777 }
778 MI.getOperand(2).setReg(Reg1);
779 MI.getOperand(1).setReg(Reg2);
780 MI.getOperand(2).setSubReg(SubReg1);
781 MI.getOperand(1).setSubReg(SubReg2);
782 MI.getOperand(2).setIsKill(Reg1IsKill);
783 MI.getOperand(1).setIsKill(Reg2IsKill);
784
785 // Swap the mask around.
786 MI.getOperand(4).setImm((ME + 1) & 31);
787 MI.getOperand(5).setImm((MB - 1) & 31);
788 return &MI;
789 }
790
findCommutedOpIndices(const MachineInstr & MI,unsigned & SrcOpIdx1,unsigned & SrcOpIdx2) const791 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
792 unsigned &SrcOpIdx1,
793 unsigned &SrcOpIdx2) const {
794 // For VSX A-Type FMA instructions, it is the first two operands that can be
795 // commuted, however, because the non-encoded tied input operand is listed
796 // first, the operands to swap are actually the second and third.
797
798 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
799 if (AltOpc == -1)
800 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
801
802 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
803 // and SrcOpIdx2.
804 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
805 }
806
insertNoop(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const807 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
808 MachineBasicBlock::iterator MI) const {
809 // This function is used for scheduling, and the nop wanted here is the type
810 // that terminates dispatch groups on the POWER cores.
811 unsigned Directive = Subtarget.getCPUDirective();
812 unsigned Opcode;
813 switch (Directive) {
814 default: Opcode = PPC::NOP; break;
815 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
816 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
817 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
818 // FIXME: Update when POWER9 scheduling model is ready.
819 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
820 }
821
822 DebugLoc DL;
823 BuildMI(MBB, MI, DL, get(Opcode));
824 }
825
826 /// Return the noop instruction to use for a noop.
getNoop(MCInst & NopInst) const827 void PPCInstrInfo::getNoop(MCInst &NopInst) const {
828 NopInst.setOpcode(PPC::NOP);
829 }
830
831 // Branch analysis.
832 // Note: If the condition register is set to CTR or CTR8 then this is a
833 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const834 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
835 MachineBasicBlock *&TBB,
836 MachineBasicBlock *&FBB,
837 SmallVectorImpl<MachineOperand> &Cond,
838 bool AllowModify) const {
839 bool isPPC64 = Subtarget.isPPC64();
840
841 // If the block has no terminators, it just falls into the block after it.
842 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
843 if (I == MBB.end())
844 return false;
845
846 if (!isUnpredicatedTerminator(*I))
847 return false;
848
849 if (AllowModify) {
850 // If the BB ends with an unconditional branch to the fallthrough BB,
851 // we eliminate the branch instruction.
852 if (I->getOpcode() == PPC::B &&
853 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
854 I->eraseFromParent();
855
856 // We update iterator after deleting the last branch.
857 I = MBB.getLastNonDebugInstr();
858 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
859 return false;
860 }
861 }
862
863 // Get the last instruction in the block.
864 MachineInstr &LastInst = *I;
865
866 // If there is only one terminator instruction, process it.
867 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
868 if (LastInst.getOpcode() == PPC::B) {
869 if (!LastInst.getOperand(0).isMBB())
870 return true;
871 TBB = LastInst.getOperand(0).getMBB();
872 return false;
873 } else if (LastInst.getOpcode() == PPC::BCC) {
874 if (!LastInst.getOperand(2).isMBB())
875 return true;
876 // Block ends with fall-through condbranch.
877 TBB = LastInst.getOperand(2).getMBB();
878 Cond.push_back(LastInst.getOperand(0));
879 Cond.push_back(LastInst.getOperand(1));
880 return false;
881 } else if (LastInst.getOpcode() == PPC::BC) {
882 if (!LastInst.getOperand(1).isMBB())
883 return true;
884 // Block ends with fall-through condbranch.
885 TBB = LastInst.getOperand(1).getMBB();
886 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
887 Cond.push_back(LastInst.getOperand(0));
888 return false;
889 } else if (LastInst.getOpcode() == PPC::BCn) {
890 if (!LastInst.getOperand(1).isMBB())
891 return true;
892 // Block ends with fall-through condbranch.
893 TBB = LastInst.getOperand(1).getMBB();
894 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
895 Cond.push_back(LastInst.getOperand(0));
896 return false;
897 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
898 LastInst.getOpcode() == PPC::BDNZ) {
899 if (!LastInst.getOperand(0).isMBB())
900 return true;
901 if (DisableCTRLoopAnal)
902 return true;
903 TBB = LastInst.getOperand(0).getMBB();
904 Cond.push_back(MachineOperand::CreateImm(1));
905 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
906 true));
907 return false;
908 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
909 LastInst.getOpcode() == PPC::BDZ) {
910 if (!LastInst.getOperand(0).isMBB())
911 return true;
912 if (DisableCTRLoopAnal)
913 return true;
914 TBB = LastInst.getOperand(0).getMBB();
915 Cond.push_back(MachineOperand::CreateImm(0));
916 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
917 true));
918 return false;
919 }
920
921 // Otherwise, don't know what this is.
922 return true;
923 }
924
925 // Get the instruction before it if it's a terminator.
926 MachineInstr &SecondLastInst = *I;
927
928 // If there are three terminators, we don't know what sort of block this is.
929 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
930 return true;
931
932 // If the block ends with PPC::B and PPC:BCC, handle it.
933 if (SecondLastInst.getOpcode() == PPC::BCC &&
934 LastInst.getOpcode() == PPC::B) {
935 if (!SecondLastInst.getOperand(2).isMBB() ||
936 !LastInst.getOperand(0).isMBB())
937 return true;
938 TBB = SecondLastInst.getOperand(2).getMBB();
939 Cond.push_back(SecondLastInst.getOperand(0));
940 Cond.push_back(SecondLastInst.getOperand(1));
941 FBB = LastInst.getOperand(0).getMBB();
942 return false;
943 } else if (SecondLastInst.getOpcode() == PPC::BC &&
944 LastInst.getOpcode() == PPC::B) {
945 if (!SecondLastInst.getOperand(1).isMBB() ||
946 !LastInst.getOperand(0).isMBB())
947 return true;
948 TBB = SecondLastInst.getOperand(1).getMBB();
949 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
950 Cond.push_back(SecondLastInst.getOperand(0));
951 FBB = LastInst.getOperand(0).getMBB();
952 return false;
953 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
954 LastInst.getOpcode() == PPC::B) {
955 if (!SecondLastInst.getOperand(1).isMBB() ||
956 !LastInst.getOperand(0).isMBB())
957 return true;
958 TBB = SecondLastInst.getOperand(1).getMBB();
959 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
960 Cond.push_back(SecondLastInst.getOperand(0));
961 FBB = LastInst.getOperand(0).getMBB();
962 return false;
963 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
964 SecondLastInst.getOpcode() == PPC::BDNZ) &&
965 LastInst.getOpcode() == PPC::B) {
966 if (!SecondLastInst.getOperand(0).isMBB() ||
967 !LastInst.getOperand(0).isMBB())
968 return true;
969 if (DisableCTRLoopAnal)
970 return true;
971 TBB = SecondLastInst.getOperand(0).getMBB();
972 Cond.push_back(MachineOperand::CreateImm(1));
973 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
974 true));
975 FBB = LastInst.getOperand(0).getMBB();
976 return false;
977 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
978 SecondLastInst.getOpcode() == PPC::BDZ) &&
979 LastInst.getOpcode() == PPC::B) {
980 if (!SecondLastInst.getOperand(0).isMBB() ||
981 !LastInst.getOperand(0).isMBB())
982 return true;
983 if (DisableCTRLoopAnal)
984 return true;
985 TBB = SecondLastInst.getOperand(0).getMBB();
986 Cond.push_back(MachineOperand::CreateImm(0));
987 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
988 true));
989 FBB = LastInst.getOperand(0).getMBB();
990 return false;
991 }
992
993 // If the block ends with two PPC:Bs, handle it. The second one is not
994 // executed, so remove it.
995 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
996 if (!SecondLastInst.getOperand(0).isMBB())
997 return true;
998 TBB = SecondLastInst.getOperand(0).getMBB();
999 I = LastInst;
1000 if (AllowModify)
1001 I->eraseFromParent();
1002 return false;
1003 }
1004
1005 // Otherwise, can't handle this.
1006 return true;
1007 }
1008
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const1009 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1010 int *BytesRemoved) const {
1011 assert(!BytesRemoved && "code size not handled");
1012
1013 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1014 if (I == MBB.end())
1015 return 0;
1016
1017 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1018 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1019 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1020 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1021 return 0;
1022
1023 // Remove the branch.
1024 I->eraseFromParent();
1025
1026 I = MBB.end();
1027
1028 if (I == MBB.begin()) return 1;
1029 --I;
1030 if (I->getOpcode() != PPC::BCC &&
1031 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1032 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1033 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1034 return 1;
1035
1036 // Remove the branch.
1037 I->eraseFromParent();
1038 return 2;
1039 }
1040
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const1041 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1042 MachineBasicBlock *TBB,
1043 MachineBasicBlock *FBB,
1044 ArrayRef<MachineOperand> Cond,
1045 const DebugLoc &DL,
1046 int *BytesAdded) const {
1047 // Shouldn't be a fall through.
1048 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1049 assert((Cond.size() == 2 || Cond.size() == 0) &&
1050 "PPC branch conditions have two components!");
1051 assert(!BytesAdded && "code size not handled");
1052
1053 bool isPPC64 = Subtarget.isPPC64();
1054
1055 // One-way branch.
1056 if (!FBB) {
1057 if (Cond.empty()) // Unconditional branch
1058 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1059 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1060 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1061 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1062 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1063 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1064 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1065 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1066 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1067 else // Conditional branch
1068 BuildMI(&MBB, DL, get(PPC::BCC))
1069 .addImm(Cond[0].getImm())
1070 .add(Cond[1])
1071 .addMBB(TBB);
1072 return 1;
1073 }
1074
1075 // Two-way Conditional Branch.
1076 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1077 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1078 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1079 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1080 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1081 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1082 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1083 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1084 else
1085 BuildMI(&MBB, DL, get(PPC::BCC))
1086 .addImm(Cond[0].getImm())
1087 .add(Cond[1])
1088 .addMBB(TBB);
1089 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1090 return 2;
1091 }
1092
1093 // Select analysis.
canInsertSelect(const MachineBasicBlock & MBB,ArrayRef<MachineOperand> Cond,Register DstReg,Register TrueReg,Register FalseReg,int & CondCycles,int & TrueCycles,int & FalseCycles) const1094 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1095 ArrayRef<MachineOperand> Cond,
1096 Register DstReg, Register TrueReg,
1097 Register FalseReg, int &CondCycles,
1098 int &TrueCycles, int &FalseCycles) const {
1099 if (Cond.size() != 2)
1100 return false;
1101
1102 // If this is really a bdnz-like condition, then it cannot be turned into a
1103 // select.
1104 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1105 return false;
1106
1107 // Check register classes.
1108 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1109 const TargetRegisterClass *RC =
1110 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1111 if (!RC)
1112 return false;
1113
1114 // isel is for regular integer GPRs only.
1115 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1116 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1117 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1118 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1119 return false;
1120
1121 // FIXME: These numbers are for the A2, how well they work for other cores is
1122 // an open question. On the A2, the isel instruction has a 2-cycle latency
1123 // but single-cycle throughput. These numbers are used in combination with
1124 // the MispredictPenalty setting from the active SchedMachineModel.
1125 CondCycles = 1;
1126 TrueCycles = 1;
1127 FalseCycles = 1;
1128
1129 return true;
1130 }
1131
insertSelect(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & dl,Register DestReg,ArrayRef<MachineOperand> Cond,Register TrueReg,Register FalseReg) const1132 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1133 MachineBasicBlock::iterator MI,
1134 const DebugLoc &dl, Register DestReg,
1135 ArrayRef<MachineOperand> Cond, Register TrueReg,
1136 Register FalseReg) const {
1137 assert(Cond.size() == 2 &&
1138 "PPC branch conditions have two components!");
1139
1140 // Get the register classes.
1141 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1142 const TargetRegisterClass *RC =
1143 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1144 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1145
1146 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1147 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1148 assert((Is64Bit ||
1149 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1150 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1151 "isel is for regular integer GPRs only");
1152
1153 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1154 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1155
1156 unsigned SubIdx = 0;
1157 bool SwapOps = false;
1158 switch (SelectPred) {
1159 case PPC::PRED_EQ:
1160 case PPC::PRED_EQ_MINUS:
1161 case PPC::PRED_EQ_PLUS:
1162 SubIdx = PPC::sub_eq; SwapOps = false; break;
1163 case PPC::PRED_NE:
1164 case PPC::PRED_NE_MINUS:
1165 case PPC::PRED_NE_PLUS:
1166 SubIdx = PPC::sub_eq; SwapOps = true; break;
1167 case PPC::PRED_LT:
1168 case PPC::PRED_LT_MINUS:
1169 case PPC::PRED_LT_PLUS:
1170 SubIdx = PPC::sub_lt; SwapOps = false; break;
1171 case PPC::PRED_GE:
1172 case PPC::PRED_GE_MINUS:
1173 case PPC::PRED_GE_PLUS:
1174 SubIdx = PPC::sub_lt; SwapOps = true; break;
1175 case PPC::PRED_GT:
1176 case PPC::PRED_GT_MINUS:
1177 case PPC::PRED_GT_PLUS:
1178 SubIdx = PPC::sub_gt; SwapOps = false; break;
1179 case PPC::PRED_LE:
1180 case PPC::PRED_LE_MINUS:
1181 case PPC::PRED_LE_PLUS:
1182 SubIdx = PPC::sub_gt; SwapOps = true; break;
1183 case PPC::PRED_UN:
1184 case PPC::PRED_UN_MINUS:
1185 case PPC::PRED_UN_PLUS:
1186 SubIdx = PPC::sub_un; SwapOps = false; break;
1187 case PPC::PRED_NU:
1188 case PPC::PRED_NU_MINUS:
1189 case PPC::PRED_NU_PLUS:
1190 SubIdx = PPC::sub_un; SwapOps = true; break;
1191 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1192 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1193 }
1194
1195 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1196 SecondReg = SwapOps ? TrueReg : FalseReg;
1197
1198 // The first input register of isel cannot be r0. If it is a member
1199 // of a register class that can be r0, then copy it first (the
1200 // register allocator should eliminate the copy).
1201 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1202 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1203 const TargetRegisterClass *FirstRC =
1204 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1205 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1206 Register OldFirstReg = FirstReg;
1207 FirstReg = MRI.createVirtualRegister(FirstRC);
1208 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1209 .addReg(OldFirstReg);
1210 }
1211
1212 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1213 .addReg(FirstReg).addReg(SecondReg)
1214 .addReg(Cond[1].getReg(), 0, SubIdx);
1215 }
1216
getCRBitValue(unsigned CRBit)1217 static unsigned getCRBitValue(unsigned CRBit) {
1218 unsigned Ret = 4;
1219 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1220 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1221 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1222 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1223 Ret = 3;
1224 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1225 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1226 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1227 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1228 Ret = 2;
1229 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1230 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1231 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1232 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1233 Ret = 1;
1234 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1235 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1236 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1237 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1238 Ret = 0;
1239
1240 assert(Ret != 4 && "Invalid CR bit register");
1241 return Ret;
1242 }
1243
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const1244 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1245 MachineBasicBlock::iterator I,
1246 const DebugLoc &DL, MCRegister DestReg,
1247 MCRegister SrcReg, bool KillSrc) const {
1248 // We can end up with self copies and similar things as a result of VSX copy
1249 // legalization. Promote them here.
1250 const TargetRegisterInfo *TRI = &getRegisterInfo();
1251 if (PPC::F8RCRegClass.contains(DestReg) &&
1252 PPC::VSRCRegClass.contains(SrcReg)) {
1253 MCRegister SuperReg =
1254 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1255
1256 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1257 llvm_unreachable("nop VSX copy");
1258
1259 DestReg = SuperReg;
1260 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1261 PPC::VSRCRegClass.contains(DestReg)) {
1262 MCRegister SuperReg =
1263 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1264
1265 if (VSXSelfCopyCrash && DestReg == SuperReg)
1266 llvm_unreachable("nop VSX copy");
1267
1268 SrcReg = SuperReg;
1269 }
1270
1271 // Different class register copy
1272 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1273 PPC::GPRCRegClass.contains(DestReg)) {
1274 MCRegister CRReg = getCRFromCRBit(SrcReg);
1275 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1276 getKillRegState(KillSrc);
1277 // Rotate the CR bit in the CR fields to be the least significant bit and
1278 // then mask with 0x1 (MB = ME = 31).
1279 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1280 .addReg(DestReg, RegState::Kill)
1281 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1282 .addImm(31)
1283 .addImm(31);
1284 return;
1285 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1286 PPC::G8RCRegClass.contains(DestReg)) {
1287 BuildMI(MBB, I, DL, get(PPC::MFOCRF8), DestReg).addReg(SrcReg);
1288 getKillRegState(KillSrc);
1289 return;
1290 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1291 PPC::GPRCRegClass.contains(DestReg)) {
1292 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(SrcReg);
1293 getKillRegState(KillSrc);
1294 return;
1295 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1296 PPC::VSFRCRegClass.contains(DestReg)) {
1297 assert(Subtarget.hasDirectMove() &&
1298 "Subtarget doesn't support directmove, don't know how to copy.");
1299 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1300 NumGPRtoVSRSpill++;
1301 getKillRegState(KillSrc);
1302 return;
1303 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1304 PPC::G8RCRegClass.contains(DestReg)) {
1305 assert(Subtarget.hasDirectMove() &&
1306 "Subtarget doesn't support directmove, don't know how to copy.");
1307 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1308 getKillRegState(KillSrc);
1309 return;
1310 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1311 PPC::GPRCRegClass.contains(DestReg)) {
1312 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1313 getKillRegState(KillSrc);
1314 return;
1315 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1316 PPC::SPERCRegClass.contains(DestReg)) {
1317 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1318 getKillRegState(KillSrc);
1319 return;
1320 }
1321
1322 unsigned Opc;
1323 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1324 Opc = PPC::OR;
1325 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1326 Opc = PPC::OR8;
1327 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1328 Opc = PPC::FMR;
1329 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1330 Opc = PPC::MCRF;
1331 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1332 Opc = PPC::VOR;
1333 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1334 // There are two different ways this can be done:
1335 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1336 // issue in VSU pipeline 0.
1337 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1338 // can go to either pipeline.
1339 // We'll always use xxlor here, because in practically all cases where
1340 // copies are generated, they are close enough to some use that the
1341 // lower-latency form is preferable.
1342 Opc = PPC::XXLOR;
1343 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1344 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1345 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1346 else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
1347 Opc = PPC::QVFMR;
1348 else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
1349 Opc = PPC::QVFMRs;
1350 else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
1351 Opc = PPC::QVFMRb;
1352 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1353 Opc = PPC::CROR;
1354 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1355 Opc = PPC::EVOR;
1356 else
1357 llvm_unreachable("Impossible reg-to-reg copy");
1358
1359 const MCInstrDesc &MCID = get(Opc);
1360 if (MCID.getNumOperands() == 3)
1361 BuildMI(MBB, I, DL, MCID, DestReg)
1362 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1363 else
1364 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1365 }
1366
getSpillIndex(const TargetRegisterClass * RC)1367 static unsigned getSpillIndex(const TargetRegisterClass *RC) {
1368 int OpcodeIndex = 0;
1369
1370 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1371 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1372 OpcodeIndex = SOK_Int4Spill;
1373 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1374 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1375 OpcodeIndex = SOK_Int8Spill;
1376 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1377 OpcodeIndex = SOK_Float8Spill;
1378 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1379 OpcodeIndex = SOK_Float4Spill;
1380 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1381 OpcodeIndex = SOK_SPESpill;
1382 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1383 OpcodeIndex = SOK_CRSpill;
1384 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1385 OpcodeIndex = SOK_CRBitSpill;
1386 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1387 OpcodeIndex = SOK_VRVectorSpill;
1388 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1389 OpcodeIndex = SOK_VSXVectorSpill;
1390 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1391 OpcodeIndex = SOK_VectorFloat8Spill;
1392 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1393 OpcodeIndex = SOK_VectorFloat4Spill;
1394 } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
1395 OpcodeIndex = SOK_VRSaveSpill;
1396 } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
1397 OpcodeIndex = SOK_QuadFloat8Spill;
1398 } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
1399 OpcodeIndex = SOK_QuadFloat4Spill;
1400 } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
1401 OpcodeIndex = SOK_QuadBitSpill;
1402 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1403 OpcodeIndex = SOK_SpillToVSR;
1404 } else {
1405 llvm_unreachable("Unknown regclass!");
1406 }
1407 return OpcodeIndex;
1408 }
1409
1410 unsigned
getStoreOpcodeForSpill(const TargetRegisterClass * RC) const1411 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1412 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1413 return OpcodesForSpill[getSpillIndex(RC)];
1414 }
1415
1416 unsigned
getLoadOpcodeForSpill(const TargetRegisterClass * RC) const1417 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1418 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1419 return OpcodesForSpill[getSpillIndex(RC)];
1420 }
1421
StoreRegToStackSlot(MachineFunction & MF,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1422 void PPCInstrInfo::StoreRegToStackSlot(
1423 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1424 const TargetRegisterClass *RC,
1425 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1426 unsigned Opcode = getStoreOpcodeForSpill(RC);
1427 DebugLoc DL;
1428
1429 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1430 FuncInfo->setHasSpills();
1431
1432 NewMIs.push_back(addFrameReference(
1433 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1434 FrameIdx));
1435
1436 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1437 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1438 FuncInfo->setSpillsCR();
1439
1440 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1441 FuncInfo->setSpillsVRSAVE();
1442
1443 if (isXFormMemOp(Opcode))
1444 FuncInfo->setHasNonRISpills();
1445 }
1446
storeRegToStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1447 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1448 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1449 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1450 const TargetRegisterInfo *TRI) const {
1451 MachineFunction &MF = *MBB.getParent();
1452 SmallVector<MachineInstr *, 4> NewMIs;
1453
1454 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1455
1456 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1457 MBB.insert(MI, NewMIs[i]);
1458
1459 const MachineFrameInfo &MFI = MF.getFrameInfo();
1460 MachineMemOperand *MMO = MF.getMachineMemOperand(
1461 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1462 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1463 MFI.getObjectAlign(FrameIdx));
1464 NewMIs.back()->addMemOperand(MF, MMO);
1465 }
1466
storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1467 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1468 MachineBasicBlock::iterator MI,
1469 Register SrcReg, bool isKill,
1470 int FrameIdx,
1471 const TargetRegisterClass *RC,
1472 const TargetRegisterInfo *TRI) const {
1473 // We need to avoid a situation in which the value from a VRRC register is
1474 // spilled using an Altivec instruction and reloaded into a VSRC register
1475 // using a VSX instruction. The issue with this is that the VSX
1476 // load/store instructions swap the doublewords in the vector and the Altivec
1477 // ones don't. The register classes on the spill/reload may be different if
1478 // the register is defined using an Altivec instruction and is then used by a
1479 // VSX instruction.
1480 RC = updatedRC(RC);
1481 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1482 }
1483
LoadRegFromStackSlot(MachineFunction & MF,const DebugLoc & DL,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1484 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1485 unsigned DestReg, int FrameIdx,
1486 const TargetRegisterClass *RC,
1487 SmallVectorImpl<MachineInstr *> &NewMIs)
1488 const {
1489 unsigned Opcode = getLoadOpcodeForSpill(RC);
1490 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1491 FrameIdx));
1492 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1493
1494 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1495 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1496 FuncInfo->setSpillsCR();
1497
1498 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1499 FuncInfo->setSpillsVRSAVE();
1500
1501 if (isXFormMemOp(Opcode))
1502 FuncInfo->setHasNonRISpills();
1503 }
1504
loadRegFromStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1505 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
1506 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
1507 int FrameIdx, const TargetRegisterClass *RC,
1508 const TargetRegisterInfo *TRI) const {
1509 MachineFunction &MF = *MBB.getParent();
1510 SmallVector<MachineInstr*, 4> NewMIs;
1511 DebugLoc DL;
1512 if (MI != MBB.end()) DL = MI->getDebugLoc();
1513
1514 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1515 FuncInfo->setHasSpills();
1516
1517 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1518
1519 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1520 MBB.insert(MI, NewMIs[i]);
1521
1522 const MachineFrameInfo &MFI = MF.getFrameInfo();
1523 MachineMemOperand *MMO = MF.getMachineMemOperand(
1524 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1525 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1526 MFI.getObjectAlign(FrameIdx));
1527 NewMIs.back()->addMemOperand(MF, MMO);
1528 }
1529
loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1530 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1531 MachineBasicBlock::iterator MI,
1532 Register DestReg, int FrameIdx,
1533 const TargetRegisterClass *RC,
1534 const TargetRegisterInfo *TRI) const {
1535 // We need to avoid a situation in which the value from a VRRC register is
1536 // spilled using an Altivec instruction and reloaded into a VSRC register
1537 // using a VSX instruction. The issue with this is that the VSX
1538 // load/store instructions swap the doublewords in the vector and the Altivec
1539 // ones don't. The register classes on the spill/reload may be different if
1540 // the register is defined using an Altivec instruction and is then used by a
1541 // VSX instruction.
1542 RC = updatedRC(RC);
1543
1544 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
1545 }
1546
1547 bool PPCInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const1548 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
1549 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
1550 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
1551 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
1552 else
1553 // Leave the CR# the same, but invert the condition.
1554 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
1555 return false;
1556 }
1557
1558 // For some instructions, it is legal to fold ZERO into the RA register field.
1559 // This function performs that fold by replacing the operand with PPC::ZERO,
1560 // it does not consider whether the load immediate zero is no longer in use.
onlyFoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg) const1561 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1562 Register Reg) const {
1563 // A zero immediate should always be loaded with a single li.
1564 unsigned DefOpc = DefMI.getOpcode();
1565 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
1566 return false;
1567 if (!DefMI.getOperand(1).isImm())
1568 return false;
1569 if (DefMI.getOperand(1).getImm() != 0)
1570 return false;
1571
1572 // Note that we cannot here invert the arguments of an isel in order to fold
1573 // a ZERO into what is presented as the second argument. All we have here
1574 // is the condition bit, and that might come from a CR-logical bit operation.
1575
1576 const MCInstrDesc &UseMCID = UseMI.getDesc();
1577
1578 // Only fold into real machine instructions.
1579 if (UseMCID.isPseudo())
1580 return false;
1581
1582 // We need to find which of the User's operands is to be folded, that will be
1583 // the operand that matches the given register ID.
1584 unsigned UseIdx;
1585 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
1586 if (UseMI.getOperand(UseIdx).isReg() &&
1587 UseMI.getOperand(UseIdx).getReg() == Reg)
1588 break;
1589
1590 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
1591 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
1592
1593 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
1594
1595 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
1596 // register (which might also be specified as a pointer class kind).
1597 if (UseInfo->isLookupPtrRegClass()) {
1598 if (UseInfo->RegClass /* Kind */ != 1)
1599 return false;
1600 } else {
1601 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
1602 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
1603 return false;
1604 }
1605
1606 // Make sure this is not tied to an output register (or otherwise
1607 // constrained). This is true for ST?UX registers, for example, which
1608 // are tied to their output registers.
1609 if (UseInfo->Constraints != 0)
1610 return false;
1611
1612 MCRegister ZeroReg;
1613 if (UseInfo->isLookupPtrRegClass()) {
1614 bool isPPC64 = Subtarget.isPPC64();
1615 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
1616 } else {
1617 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
1618 PPC::ZERO8 : PPC::ZERO;
1619 }
1620
1621 UseMI.getOperand(UseIdx).setReg(ZeroReg);
1622 return true;
1623 }
1624
1625 // Folds zero into instructions which have a load immediate zero as an operand
1626 // but also recognize zero as immediate zero. If the definition of the load
1627 // has no more users it is deleted.
FoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg,MachineRegisterInfo * MRI) const1628 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1629 Register Reg, MachineRegisterInfo *MRI) const {
1630 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
1631 if (MRI->use_nodbg_empty(Reg))
1632 DefMI.eraseFromParent();
1633 return Changed;
1634 }
1635
MBBDefinesCTR(MachineBasicBlock & MBB)1636 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
1637 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
1638 I != IE; ++I)
1639 if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8))
1640 return true;
1641 return false;
1642 }
1643
1644 // We should make sure that, if we're going to predicate both sides of a
1645 // condition (a diamond), that both sides don't define the counter register. We
1646 // can predicate counter-decrement-based branches, but while that predicates
1647 // the branching, it does not predicate the counter decrement. If we tried to
1648 // merge the triangle into one predicated block, we'd decrement the counter
1649 // twice.
isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumT,unsigned ExtraT,MachineBasicBlock & FMBB,unsigned NumF,unsigned ExtraF,BranchProbability Probability) const1650 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
1651 unsigned NumT, unsigned ExtraT,
1652 MachineBasicBlock &FMBB,
1653 unsigned NumF, unsigned ExtraF,
1654 BranchProbability Probability) const {
1655 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
1656 }
1657
1658
isPredicated(const MachineInstr & MI) const1659 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
1660 // The predicated branches are identified by their type, not really by the
1661 // explicit presence of a predicate. Furthermore, some of them can be
1662 // predicated more than once. Because if conversion won't try to predicate
1663 // any instruction which already claims to be predicated (by returning true
1664 // here), always return false. In doing so, we let isPredicable() be the
1665 // final word on whether not the instruction can be (further) predicated.
1666
1667 return false;
1668 }
1669
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const1670 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
1671 ArrayRef<MachineOperand> Pred) const {
1672 unsigned OpC = MI.getOpcode();
1673 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
1674 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1675 bool isPPC64 = Subtarget.isPPC64();
1676 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
1677 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
1678 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1679 MI.setDesc(get(PPC::BCLR));
1680 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1681 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1682 MI.setDesc(get(PPC::BCLRn));
1683 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1684 } else {
1685 MI.setDesc(get(PPC::BCCLR));
1686 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1687 .addImm(Pred[0].getImm())
1688 .add(Pred[1]);
1689 }
1690
1691 return true;
1692 } else if (OpC == PPC::B) {
1693 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1694 bool isPPC64 = Subtarget.isPPC64();
1695 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
1696 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
1697 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1698 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1699 MI.RemoveOperand(0);
1700
1701 MI.setDesc(get(PPC::BC));
1702 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1703 .add(Pred[1])
1704 .addMBB(MBB);
1705 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1706 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1707 MI.RemoveOperand(0);
1708
1709 MI.setDesc(get(PPC::BCn));
1710 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1711 .add(Pred[1])
1712 .addMBB(MBB);
1713 } else {
1714 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1715 MI.RemoveOperand(0);
1716
1717 MI.setDesc(get(PPC::BCC));
1718 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1719 .addImm(Pred[0].getImm())
1720 .add(Pred[1])
1721 .addMBB(MBB);
1722 }
1723
1724 return true;
1725 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
1726 OpC == PPC::BCTRL8) {
1727 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
1728 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
1729
1730 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8;
1731 bool isPPC64 = Subtarget.isPPC64();
1732
1733 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1734 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
1735 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
1736 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1737 return true;
1738 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1739 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
1740 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
1741 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1742 return true;
1743 }
1744
1745 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
1746 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
1747 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1748 .addImm(Pred[0].getImm())
1749 .add(Pred[1]);
1750 return true;
1751 }
1752
1753 return false;
1754 }
1755
SubsumesPredicate(ArrayRef<MachineOperand> Pred1,ArrayRef<MachineOperand> Pred2) const1756 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
1757 ArrayRef<MachineOperand> Pred2) const {
1758 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
1759 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
1760
1761 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
1762 return false;
1763 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
1764 return false;
1765
1766 // P1 can only subsume P2 if they test the same condition register.
1767 if (Pred1[1].getReg() != Pred2[1].getReg())
1768 return false;
1769
1770 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
1771 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
1772
1773 if (P1 == P2)
1774 return true;
1775
1776 // Does P1 subsume P2, e.g. GE subsumes GT.
1777 if (P1 == PPC::PRED_LE &&
1778 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
1779 return true;
1780 if (P1 == PPC::PRED_GE &&
1781 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
1782 return true;
1783
1784 return false;
1785 }
1786
DefinesPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred) const1787 bool PPCInstrInfo::DefinesPredicate(MachineInstr &MI,
1788 std::vector<MachineOperand> &Pred) const {
1789 // Note: At the present time, the contents of Pred from this function is
1790 // unused by IfConversion. This implementation follows ARM by pushing the
1791 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
1792 // predicate, instructions defining CTR or CTR8 are also included as
1793 // predicate-defining instructions.
1794
1795 const TargetRegisterClass *RCs[] =
1796 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
1797 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
1798
1799 bool Found = false;
1800 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1801 const MachineOperand &MO = MI.getOperand(i);
1802 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
1803 const TargetRegisterClass *RC = RCs[c];
1804 if (MO.isReg()) {
1805 if (MO.isDef() && RC->contains(MO.getReg())) {
1806 Pred.push_back(MO);
1807 Found = true;
1808 }
1809 } else if (MO.isRegMask()) {
1810 for (TargetRegisterClass::iterator I = RC->begin(),
1811 IE = RC->end(); I != IE; ++I)
1812 if (MO.clobbersPhysReg(*I)) {
1813 Pred.push_back(MO);
1814 Found = true;
1815 }
1816 }
1817 }
1818 }
1819
1820 return Found;
1821 }
1822
analyzeCompare(const MachineInstr & MI,Register & SrcReg,Register & SrcReg2,int & Mask,int & Value) const1823 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1824 Register &SrcReg2, int &Mask,
1825 int &Value) const {
1826 unsigned Opc = MI.getOpcode();
1827
1828 switch (Opc) {
1829 default: return false;
1830 case PPC::CMPWI:
1831 case PPC::CMPLWI:
1832 case PPC::CMPDI:
1833 case PPC::CMPLDI:
1834 SrcReg = MI.getOperand(1).getReg();
1835 SrcReg2 = 0;
1836 Value = MI.getOperand(2).getImm();
1837 Mask = 0xFFFF;
1838 return true;
1839 case PPC::CMPW:
1840 case PPC::CMPLW:
1841 case PPC::CMPD:
1842 case PPC::CMPLD:
1843 case PPC::FCMPUS:
1844 case PPC::FCMPUD:
1845 SrcReg = MI.getOperand(1).getReg();
1846 SrcReg2 = MI.getOperand(2).getReg();
1847 Value = 0;
1848 Mask = 0;
1849 return true;
1850 }
1851 }
1852
optimizeCompareInstr(MachineInstr & CmpInstr,Register SrcReg,Register SrcReg2,int Mask,int Value,const MachineRegisterInfo * MRI) const1853 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
1854 Register SrcReg2, int Mask, int Value,
1855 const MachineRegisterInfo *MRI) const {
1856 if (DisableCmpOpt)
1857 return false;
1858
1859 int OpC = CmpInstr.getOpcode();
1860 Register CRReg = CmpInstr.getOperand(0).getReg();
1861
1862 // FP record forms set CR1 based on the exception status bits, not a
1863 // comparison with zero.
1864 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
1865 return false;
1866
1867 const TargetRegisterInfo *TRI = &getRegisterInfo();
1868 // The record forms set the condition register based on a signed comparison
1869 // with zero (so says the ISA manual). This is not as straightforward as it
1870 // seems, however, because this is always a 64-bit comparison on PPC64, even
1871 // for instructions that are 32-bit in nature (like slw for example).
1872 // So, on PPC32, for unsigned comparisons, we can use the record forms only
1873 // for equality checks (as those don't depend on the sign). On PPC64,
1874 // we are restricted to equality for unsigned 64-bit comparisons and for
1875 // signed 32-bit comparisons the applicability is more restricted.
1876 bool isPPC64 = Subtarget.isPPC64();
1877 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
1878 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
1879 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
1880
1881 // Look through copies unless that gets us to a physical register.
1882 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
1883 if (ActualSrc.isVirtual())
1884 SrcReg = ActualSrc;
1885
1886 // Get the unique definition of SrcReg.
1887 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1888 if (!MI) return false;
1889
1890 bool equalityOnly = false;
1891 bool noSub = false;
1892 if (isPPC64) {
1893 if (is32BitSignedCompare) {
1894 // We can perform this optimization only if MI is sign-extending.
1895 if (isSignExtended(*MI))
1896 noSub = true;
1897 else
1898 return false;
1899 } else if (is32BitUnsignedCompare) {
1900 // We can perform this optimization, equality only, if MI is
1901 // zero-extending.
1902 if (isZeroExtended(*MI)) {
1903 noSub = true;
1904 equalityOnly = true;
1905 } else
1906 return false;
1907 } else
1908 equalityOnly = is64BitUnsignedCompare;
1909 } else
1910 equalityOnly = is32BitUnsignedCompare;
1911
1912 if (equalityOnly) {
1913 // We need to check the uses of the condition register in order to reject
1914 // non-equality comparisons.
1915 for (MachineRegisterInfo::use_instr_iterator
1916 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
1917 I != IE; ++I) {
1918 MachineInstr *UseMI = &*I;
1919 if (UseMI->getOpcode() == PPC::BCC) {
1920 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1921 unsigned PredCond = PPC::getPredicateCondition(Pred);
1922 // We ignore hint bits when checking for non-equality comparisons.
1923 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
1924 return false;
1925 } else if (UseMI->getOpcode() == PPC::ISEL ||
1926 UseMI->getOpcode() == PPC::ISEL8) {
1927 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
1928 if (SubIdx != PPC::sub_eq)
1929 return false;
1930 } else
1931 return false;
1932 }
1933 }
1934
1935 MachineBasicBlock::iterator I = CmpInstr;
1936
1937 // Scan forward to find the first use of the compare.
1938 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
1939 ++I) {
1940 bool FoundUse = false;
1941 for (MachineRegisterInfo::use_instr_iterator
1942 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
1943 J != JE; ++J)
1944 if (&*J == &*I) {
1945 FoundUse = true;
1946 break;
1947 }
1948
1949 if (FoundUse)
1950 break;
1951 }
1952
1953 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
1954 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
1955
1956 // There are two possible candidates which can be changed to set CR[01].
1957 // One is MI, the other is a SUB instruction.
1958 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
1959 MachineInstr *Sub = nullptr;
1960 if (SrcReg2 != 0)
1961 // MI is not a candidate for CMPrr.
1962 MI = nullptr;
1963 // FIXME: Conservatively refuse to convert an instruction which isn't in the
1964 // same BB as the comparison. This is to allow the check below to avoid calls
1965 // (and other explicit clobbers); instead we should really check for these
1966 // more explicitly (in at least a few predecessors).
1967 else if (MI->getParent() != CmpInstr.getParent())
1968 return false;
1969 else if (Value != 0) {
1970 // The record-form instructions set CR bit based on signed comparison
1971 // against 0. We try to convert a compare against 1 or -1 into a compare
1972 // against 0 to exploit record-form instructions. For example, we change
1973 // the condition "greater than -1" into "greater than or equal to 0"
1974 // and "less than 1" into "less than or equal to 0".
1975
1976 // Since we optimize comparison based on a specific branch condition,
1977 // we don't optimize if condition code is used by more than once.
1978 if (equalityOnly || !MRI->hasOneUse(CRReg))
1979 return false;
1980
1981 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
1982 if (UseMI->getOpcode() != PPC::BCC)
1983 return false;
1984
1985 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1986 unsigned PredCond = PPC::getPredicateCondition(Pred);
1987 unsigned PredHint = PPC::getPredicateHint(Pred);
1988 int16_t Immed = (int16_t)Value;
1989
1990 // When modifying the condition in the predicate, we propagate hint bits
1991 // from the original predicate to the new one.
1992 if (Immed == -1 && PredCond == PPC::PRED_GT)
1993 // We convert "greater than -1" into "greater than or equal to 0",
1994 // since we are assuming signed comparison by !equalityOnly
1995 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
1996 else if (Immed == -1 && PredCond == PPC::PRED_LE)
1997 // We convert "less than or equal to -1" into "less than 0".
1998 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
1999 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2000 // We convert "less than 1" into "less than or equal to 0".
2001 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2002 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2003 // We convert "greater than or equal to 1" into "greater than 0".
2004 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2005 else
2006 return false;
2007
2008 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
2009 }
2010
2011 // Search for Sub.
2012 --I;
2013
2014 // Get ready to iterate backward from CmpInstr.
2015 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2016
2017 for (; I != E && !noSub; --I) {
2018 const MachineInstr &Instr = *I;
2019 unsigned IOpC = Instr.getOpcode();
2020
2021 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2022 Instr.readsRegister(PPC::CR0, TRI)))
2023 // This instruction modifies or uses the record condition register after
2024 // the one we want to change. While we could do this transformation, it
2025 // would likely not be profitable. This transformation removes one
2026 // instruction, and so even forcing RA to generate one move probably
2027 // makes it unprofitable.
2028 return false;
2029
2030 // Check whether CmpInstr can be made redundant by the current instruction.
2031 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2032 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2033 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2034 ((Instr.getOperand(1).getReg() == SrcReg &&
2035 Instr.getOperand(2).getReg() == SrcReg2) ||
2036 (Instr.getOperand(1).getReg() == SrcReg2 &&
2037 Instr.getOperand(2).getReg() == SrcReg))) {
2038 Sub = &*I;
2039 break;
2040 }
2041
2042 if (I == B)
2043 // The 'and' is below the comparison instruction.
2044 return false;
2045 }
2046
2047 // Return false if no candidates exist.
2048 if (!MI && !Sub)
2049 return false;
2050
2051 // The single candidate is called MI.
2052 if (!MI) MI = Sub;
2053
2054 int NewOpC = -1;
2055 int MIOpC = MI->getOpcode();
2056 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2057 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2058 NewOpC = MIOpC;
2059 else {
2060 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2061 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2062 NewOpC = MIOpC;
2063 }
2064
2065 // FIXME: On the non-embedded POWER architectures, only some of the record
2066 // forms are fast, and we should use only the fast ones.
2067
2068 // The defining instruction has a record form (or is already a record
2069 // form). It is possible, however, that we'll need to reverse the condition
2070 // code of the users.
2071 if (NewOpC == -1)
2072 return false;
2073
2074 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2075 // needs to be updated to be based on SUB. Push the condition code
2076 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2077 // condition code of these operands will be modified.
2078 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2079 // comparison against 0, which may modify predicate.
2080 bool ShouldSwap = false;
2081 if (Sub && Value == 0) {
2082 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2083 Sub->getOperand(2).getReg() == SrcReg;
2084
2085 // The operands to subf are the opposite of sub, so only in the fixed-point
2086 // case, invert the order.
2087 ShouldSwap = !ShouldSwap;
2088 }
2089
2090 if (ShouldSwap)
2091 for (MachineRegisterInfo::use_instr_iterator
2092 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2093 I != IE; ++I) {
2094 MachineInstr *UseMI = &*I;
2095 if (UseMI->getOpcode() == PPC::BCC) {
2096 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2097 unsigned PredCond = PPC::getPredicateCondition(Pred);
2098 assert((!equalityOnly ||
2099 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2100 "Invalid predicate for equality-only optimization");
2101 (void)PredCond; // To suppress warning in release build.
2102 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2103 PPC::getSwappedPredicate(Pred)));
2104 } else if (UseMI->getOpcode() == PPC::ISEL ||
2105 UseMI->getOpcode() == PPC::ISEL8) {
2106 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2107 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2108 "Invalid CR bit for equality-only optimization");
2109
2110 if (NewSubReg == PPC::sub_lt)
2111 NewSubReg = PPC::sub_gt;
2112 else if (NewSubReg == PPC::sub_gt)
2113 NewSubReg = PPC::sub_lt;
2114
2115 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2116 NewSubReg));
2117 } else // We need to abort on a user we don't understand.
2118 return false;
2119 }
2120 assert(!(Value != 0 && ShouldSwap) &&
2121 "Non-zero immediate support and ShouldSwap"
2122 "may conflict in updating predicate");
2123
2124 // Create a new virtual register to hold the value of the CR set by the
2125 // record-form instruction. If the instruction was not previously in
2126 // record form, then set the kill flag on the CR.
2127 CmpInstr.eraseFromParent();
2128
2129 MachineBasicBlock::iterator MII = MI;
2130 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2131 get(TargetOpcode::COPY), CRReg)
2132 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2133
2134 // Even if CR0 register were dead before, it is alive now since the
2135 // instruction we just built uses it.
2136 MI->clearRegisterDeads(PPC::CR0);
2137
2138 if (MIOpC != NewOpC) {
2139 // We need to be careful here: we're replacing one instruction with
2140 // another, and we need to make sure that we get all of the right
2141 // implicit uses and defs. On the other hand, the caller may be holding
2142 // an iterator to this instruction, and so we can't delete it (this is
2143 // specifically the case if this is the instruction directly after the
2144 // compare).
2145
2146 // Rotates are expensive instructions. If we're emitting a record-form
2147 // rotate that can just be an andi/andis, we should just emit that.
2148 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2149 Register GPRRes = MI->getOperand(0).getReg();
2150 int64_t SH = MI->getOperand(2).getImm();
2151 int64_t MB = MI->getOperand(3).getImm();
2152 int64_t ME = MI->getOperand(4).getImm();
2153 // We can only do this if both the start and end of the mask are in the
2154 // same halfword.
2155 bool MBInLoHWord = MB >= 16;
2156 bool MEInLoHWord = ME >= 16;
2157 uint64_t Mask = ~0LLU;
2158
2159 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2160 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2161 // The mask value needs to shift right 16 if we're emitting andis.
2162 Mask >>= MBInLoHWord ? 0 : 16;
2163 NewOpC = MIOpC == PPC::RLWINM
2164 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2165 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2166 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2167 (ME - MB + 1 == SH) && (MB >= 16)) {
2168 // If we are rotating by the exact number of bits as are in the mask
2169 // and the mask is in the least significant bits of the register,
2170 // that's just an andis. (as long as the GPR result has no uses).
2171 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2172 Mask >>= 16;
2173 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2174 }
2175 // If we've set the mask, we can transform.
2176 if (Mask != ~0LLU) {
2177 MI->RemoveOperand(4);
2178 MI->RemoveOperand(3);
2179 MI->getOperand(2).setImm(Mask);
2180 NumRcRotatesConvertedToRcAnd++;
2181 }
2182 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2183 int64_t MB = MI->getOperand(3).getImm();
2184 if (MB >= 48) {
2185 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2186 NewOpC = PPC::ANDI8_rec;
2187 MI->RemoveOperand(3);
2188 MI->getOperand(2).setImm(Mask);
2189 NumRcRotatesConvertedToRcAnd++;
2190 }
2191 }
2192
2193 const MCInstrDesc &NewDesc = get(NewOpC);
2194 MI->setDesc(NewDesc);
2195
2196 if (NewDesc.ImplicitDefs)
2197 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2198 *ImpDefs; ++ImpDefs)
2199 if (!MI->definesRegister(*ImpDefs))
2200 MI->addOperand(*MI->getParent()->getParent(),
2201 MachineOperand::CreateReg(*ImpDefs, true, true));
2202 if (NewDesc.ImplicitUses)
2203 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2204 *ImpUses; ++ImpUses)
2205 if (!MI->readsRegister(*ImpUses))
2206 MI->addOperand(*MI->getParent()->getParent(),
2207 MachineOperand::CreateReg(*ImpUses, false, true));
2208 }
2209 assert(MI->definesRegister(PPC::CR0) &&
2210 "Record-form instruction does not define cr0?");
2211
2212 // Modify the condition code of operands in OperandsToUpdate.
2213 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2214 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2215 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2216 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2217
2218 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2219 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2220
2221 return true;
2222 }
2223
2224 /// GetInstSize - Return the number of bytes of code the specified
2225 /// instruction may be. This returns the maximum number of bytes.
2226 ///
getInstSizeInBytes(const MachineInstr & MI) const2227 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2228 unsigned Opcode = MI.getOpcode();
2229
2230 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2231 const MachineFunction *MF = MI.getParent()->getParent();
2232 const char *AsmStr = MI.getOperand(0).getSymbolName();
2233 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2234 } else if (Opcode == TargetOpcode::STACKMAP) {
2235 StackMapOpers Opers(&MI);
2236 return Opers.getNumPatchBytes();
2237 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2238 PatchPointOpers Opers(&MI);
2239 return Opers.getNumPatchBytes();
2240 } else {
2241 return get(Opcode).getSize();
2242 }
2243 }
2244
2245 std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const2246 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2247 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2248 return std::make_pair(TF & Mask, TF & ~Mask);
2249 }
2250
2251 ArrayRef<std::pair<unsigned, const char *>>
getSerializableDirectMachineOperandTargetFlags() const2252 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2253 using namespace PPCII;
2254 static const std::pair<unsigned, const char *> TargetFlags[] = {
2255 {MO_LO, "ppc-lo"},
2256 {MO_HA, "ppc-ha"},
2257 {MO_TPREL_LO, "ppc-tprel-lo"},
2258 {MO_TPREL_HA, "ppc-tprel-ha"},
2259 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2260 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2261 {MO_TOC_LO, "ppc-toc-lo"},
2262 {MO_TLS, "ppc-tls"}};
2263 return makeArrayRef(TargetFlags);
2264 }
2265
2266 ArrayRef<std::pair<unsigned, const char *>>
getSerializableBitmaskMachineOperandTargetFlags() const2267 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
2268 using namespace PPCII;
2269 static const std::pair<unsigned, const char *> TargetFlags[] = {
2270 {MO_PLT, "ppc-plt"},
2271 {MO_PIC_FLAG, "ppc-pic"},
2272 {MO_PCREL_FLAG, "ppc-pcrel"},
2273 {MO_GOT_FLAG, "ppc-got"}};
2274 return makeArrayRef(TargetFlags);
2275 }
2276
2277 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2278 // The VSX versions have the advantage of a full 64-register target whereas
2279 // the FP ones have the advantage of lower latency and higher throughput. So
2280 // what we are after is using the faster instructions in low register pressure
2281 // situations and using the larger register file in high register pressure
2282 // situations.
expandVSXMemPseudo(MachineInstr & MI) const2283 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2284 unsigned UpperOpcode, LowerOpcode;
2285 switch (MI.getOpcode()) {
2286 case PPC::DFLOADf32:
2287 UpperOpcode = PPC::LXSSP;
2288 LowerOpcode = PPC::LFS;
2289 break;
2290 case PPC::DFLOADf64:
2291 UpperOpcode = PPC::LXSD;
2292 LowerOpcode = PPC::LFD;
2293 break;
2294 case PPC::DFSTOREf32:
2295 UpperOpcode = PPC::STXSSP;
2296 LowerOpcode = PPC::STFS;
2297 break;
2298 case PPC::DFSTOREf64:
2299 UpperOpcode = PPC::STXSD;
2300 LowerOpcode = PPC::STFD;
2301 break;
2302 case PPC::XFLOADf32:
2303 UpperOpcode = PPC::LXSSPX;
2304 LowerOpcode = PPC::LFSX;
2305 break;
2306 case PPC::XFLOADf64:
2307 UpperOpcode = PPC::LXSDX;
2308 LowerOpcode = PPC::LFDX;
2309 break;
2310 case PPC::XFSTOREf32:
2311 UpperOpcode = PPC::STXSSPX;
2312 LowerOpcode = PPC::STFSX;
2313 break;
2314 case PPC::XFSTOREf64:
2315 UpperOpcode = PPC::STXSDX;
2316 LowerOpcode = PPC::STFDX;
2317 break;
2318 case PPC::LIWAX:
2319 UpperOpcode = PPC::LXSIWAX;
2320 LowerOpcode = PPC::LFIWAX;
2321 break;
2322 case PPC::LIWZX:
2323 UpperOpcode = PPC::LXSIWZX;
2324 LowerOpcode = PPC::LFIWZX;
2325 break;
2326 case PPC::STIWX:
2327 UpperOpcode = PPC::STXSIWX;
2328 LowerOpcode = PPC::STFIWX;
2329 break;
2330 default:
2331 llvm_unreachable("Unknown Operation!");
2332 }
2333
2334 Register TargetReg = MI.getOperand(0).getReg();
2335 unsigned Opcode;
2336 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2337 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2338 Opcode = LowerOpcode;
2339 else
2340 Opcode = UpperOpcode;
2341 MI.setDesc(get(Opcode));
2342 return true;
2343 }
2344
isAnImmediateOperand(const MachineOperand & MO)2345 static bool isAnImmediateOperand(const MachineOperand &MO) {
2346 return MO.isCPI() || MO.isGlobal() || MO.isImm();
2347 }
2348
expandPostRAPseudo(MachineInstr & MI) const2349 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2350 auto &MBB = *MI.getParent();
2351 auto DL = MI.getDebugLoc();
2352
2353 switch (MI.getOpcode()) {
2354 case TargetOpcode::LOAD_STACK_GUARD: {
2355 assert(Subtarget.isTargetLinux() &&
2356 "Only Linux target is expected to contain LOAD_STACK_GUARD");
2357 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
2358 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
2359 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
2360 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2361 .addImm(Offset)
2362 .addReg(Reg);
2363 return true;
2364 }
2365 case PPC::DFLOADf32:
2366 case PPC::DFLOADf64:
2367 case PPC::DFSTOREf32:
2368 case PPC::DFSTOREf64: {
2369 assert(Subtarget.hasP9Vector() &&
2370 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
2371 assert(MI.getOperand(2).isReg() &&
2372 isAnImmediateOperand(MI.getOperand(1)) &&
2373 "D-form op must have register and immediate operands");
2374 return expandVSXMemPseudo(MI);
2375 }
2376 case PPC::XFLOADf32:
2377 case PPC::XFSTOREf32:
2378 case PPC::LIWAX:
2379 case PPC::LIWZX:
2380 case PPC::STIWX: {
2381 assert(Subtarget.hasP8Vector() &&
2382 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
2383 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2384 "X-form op must have register and register operands");
2385 return expandVSXMemPseudo(MI);
2386 }
2387 case PPC::XFLOADf64:
2388 case PPC::XFSTOREf64: {
2389 assert(Subtarget.hasVSX() &&
2390 "Invalid X-Form Pseudo-ops on target that has no VSX.");
2391 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2392 "X-form op must have register and register operands");
2393 return expandVSXMemPseudo(MI);
2394 }
2395 case PPC::SPILLTOVSR_LD: {
2396 Register TargetReg = MI.getOperand(0).getReg();
2397 if (PPC::VSFRCRegClass.contains(TargetReg)) {
2398 MI.setDesc(get(PPC::DFLOADf64));
2399 return expandPostRAPseudo(MI);
2400 }
2401 else
2402 MI.setDesc(get(PPC::LD));
2403 return true;
2404 }
2405 case PPC::SPILLTOVSR_ST: {
2406 Register SrcReg = MI.getOperand(0).getReg();
2407 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2408 NumStoreSPILLVSRRCAsVec++;
2409 MI.setDesc(get(PPC::DFSTOREf64));
2410 return expandPostRAPseudo(MI);
2411 } else {
2412 NumStoreSPILLVSRRCAsGpr++;
2413 MI.setDesc(get(PPC::STD));
2414 }
2415 return true;
2416 }
2417 case PPC::SPILLTOVSR_LDX: {
2418 Register TargetReg = MI.getOperand(0).getReg();
2419 if (PPC::VSFRCRegClass.contains(TargetReg))
2420 MI.setDesc(get(PPC::LXSDX));
2421 else
2422 MI.setDesc(get(PPC::LDX));
2423 return true;
2424 }
2425 case PPC::SPILLTOVSR_STX: {
2426 Register SrcReg = MI.getOperand(0).getReg();
2427 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2428 NumStoreSPILLVSRRCAsVec++;
2429 MI.setDesc(get(PPC::STXSDX));
2430 } else {
2431 NumStoreSPILLVSRRCAsGpr++;
2432 MI.setDesc(get(PPC::STDX));
2433 }
2434 return true;
2435 }
2436
2437 case PPC::CFENCE8: {
2438 auto Val = MI.getOperand(0).getReg();
2439 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
2440 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
2441 .addImm(PPC::PRED_NE_MINUS)
2442 .addReg(PPC::CR7)
2443 .addImm(1);
2444 MI.setDesc(get(PPC::ISYNC));
2445 MI.RemoveOperand(0);
2446 return true;
2447 }
2448 }
2449 return false;
2450 }
2451
2452 // Essentially a compile-time implementation of a compare->isel sequence.
2453 // It takes two constants to compare, along with the true/false registers
2454 // and the comparison type (as a subreg to a CR field) and returns one
2455 // of the true/false registers, depending on the comparison results.
selectReg(int64_t Imm1,int64_t Imm2,unsigned CompareOpc,unsigned TrueReg,unsigned FalseReg,unsigned CRSubReg)2456 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
2457 unsigned TrueReg, unsigned FalseReg,
2458 unsigned CRSubReg) {
2459 // Signed comparisons. The immediates are assumed to be sign-extended.
2460 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
2461 switch (CRSubReg) {
2462 default: llvm_unreachable("Unknown integer comparison type.");
2463 case PPC::sub_lt:
2464 return Imm1 < Imm2 ? TrueReg : FalseReg;
2465 case PPC::sub_gt:
2466 return Imm1 > Imm2 ? TrueReg : FalseReg;
2467 case PPC::sub_eq:
2468 return Imm1 == Imm2 ? TrueReg : FalseReg;
2469 }
2470 }
2471 // Unsigned comparisons.
2472 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
2473 switch (CRSubReg) {
2474 default: llvm_unreachable("Unknown integer comparison type.");
2475 case PPC::sub_lt:
2476 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
2477 case PPC::sub_gt:
2478 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
2479 case PPC::sub_eq:
2480 return Imm1 == Imm2 ? TrueReg : FalseReg;
2481 }
2482 }
2483 return PPC::NoRegister;
2484 }
2485
replaceInstrOperandWithImm(MachineInstr & MI,unsigned OpNo,int64_t Imm) const2486 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
2487 unsigned OpNo,
2488 int64_t Imm) const {
2489 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
2490 // Replace the REG with the Immediate.
2491 Register InUseReg = MI.getOperand(OpNo).getReg();
2492 MI.getOperand(OpNo).ChangeToImmediate(Imm);
2493
2494 if (MI.implicit_operands().empty())
2495 return;
2496
2497 // We need to make sure that the MI didn't have any implicit use
2498 // of this REG any more.
2499 const TargetRegisterInfo *TRI = &getRegisterInfo();
2500 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
2501 if (UseOpIdx >= 0) {
2502 MachineOperand &MO = MI.getOperand(UseOpIdx);
2503 if (MO.isImplicit())
2504 // The operands must always be in the following order:
2505 // - explicit reg defs,
2506 // - other explicit operands (reg uses, immediates, etc.),
2507 // - implicit reg defs
2508 // - implicit reg uses
2509 // Therefore, removing the implicit operand won't change the explicit
2510 // operands layout.
2511 MI.RemoveOperand(UseOpIdx);
2512 }
2513 }
2514
2515 // Replace an instruction with one that materializes a constant (and sets
2516 // CR0 if the original instruction was a record-form instruction).
replaceInstrWithLI(MachineInstr & MI,const LoadImmediateInfo & LII) const2517 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
2518 const LoadImmediateInfo &LII) const {
2519 // Remove existing operands.
2520 int OperandToKeep = LII.SetCR ? 1 : 0;
2521 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
2522 MI.RemoveOperand(i);
2523
2524 // Replace the instruction.
2525 if (LII.SetCR) {
2526 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
2527 // Set the immediate.
2528 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2529 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
2530 return;
2531 }
2532 else
2533 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
2534
2535 // Set the immediate.
2536 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2537 .addImm(LII.Imm);
2538 }
2539
getDefMIPostRA(unsigned Reg,MachineInstr & MI,bool & SeenIntermediateUse) const2540 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
2541 bool &SeenIntermediateUse) const {
2542 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
2543 "Should be called after register allocation.");
2544 const TargetRegisterInfo *TRI = &getRegisterInfo();
2545 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
2546 It++;
2547 SeenIntermediateUse = false;
2548 for (; It != E; ++It) {
2549 if (It->modifiesRegister(Reg, TRI))
2550 return &*It;
2551 if (It->readsRegister(Reg, TRI))
2552 SeenIntermediateUse = true;
2553 }
2554 return nullptr;
2555 }
2556
getForwardingDefMI(MachineInstr & MI,unsigned & OpNoForForwarding,bool & SeenIntermediateUse) const2557 MachineInstr *PPCInstrInfo::getForwardingDefMI(
2558 MachineInstr &MI,
2559 unsigned &OpNoForForwarding,
2560 bool &SeenIntermediateUse) const {
2561 OpNoForForwarding = ~0U;
2562 MachineInstr *DefMI = nullptr;
2563 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
2564 const TargetRegisterInfo *TRI = &getRegisterInfo();
2565 // If we're in SSA, get the defs through the MRI. Otherwise, only look
2566 // within the basic block to see if the register is defined using an
2567 // LI/LI8/ADDI/ADDI8.
2568 if (MRI->isSSA()) {
2569 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2570 if (!MI.getOperand(i).isReg())
2571 continue;
2572 Register Reg = MI.getOperand(i).getReg();
2573 if (!Register::isVirtualRegister(Reg))
2574 continue;
2575 unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
2576 if (Register::isVirtualRegister(TrueReg)) {
2577 DefMI = MRI->getVRegDef(TrueReg);
2578 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
2579 DefMI->getOpcode() == PPC::ADDI ||
2580 DefMI->getOpcode() == PPC::ADDI8) {
2581 OpNoForForwarding = i;
2582 // The ADDI and LI operand maybe exist in one instruction at same
2583 // time. we prefer to fold LI operand as LI only has one Imm operand
2584 // and is more possible to be converted. So if current DefMI is
2585 // ADDI/ADDI8, we continue to find possible LI/LI8.
2586 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
2587 break;
2588 }
2589 }
2590 }
2591 } else {
2592 // Looking back through the definition for each operand could be expensive,
2593 // so exit early if this isn't an instruction that either has an immediate
2594 // form or is already an immediate form that we can handle.
2595 ImmInstrInfo III;
2596 unsigned Opc = MI.getOpcode();
2597 bool ConvertibleImmForm =
2598 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
2599 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
2600 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
2601 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
2602 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
2603 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
2604 Opc == PPC::RLWINM8_rec;
2605 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
2606 ? isVFRegister(MI.getOperand(0).getReg())
2607 : false;
2608 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
2609 return nullptr;
2610
2611 // Don't convert or %X, %Y, %Y since that's just a register move.
2612 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
2613 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2614 return nullptr;
2615 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2616 MachineOperand &MO = MI.getOperand(i);
2617 SeenIntermediateUse = false;
2618 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
2619 Register Reg = MI.getOperand(i).getReg();
2620 // If we see another use of this reg between the def and the MI,
2621 // we want to flat it so the def isn't deleted.
2622 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
2623 if (DefMI) {
2624 // Is this register defined by some form of add-immediate (including
2625 // load-immediate) within this basic block?
2626 switch (DefMI->getOpcode()) {
2627 default:
2628 break;
2629 case PPC::LI:
2630 case PPC::LI8:
2631 case PPC::ADDItocL:
2632 case PPC::ADDI:
2633 case PPC::ADDI8:
2634 OpNoForForwarding = i;
2635 return DefMI;
2636 }
2637 }
2638 }
2639 }
2640 }
2641 return OpNoForForwarding == ~0U ? nullptr : DefMI;
2642 }
2643
getSpillTarget() const2644 unsigned PPCInstrInfo::getSpillTarget() const {
2645 return Subtarget.hasP9Vector() ? 1 : 0;
2646 }
2647
getStoreOpcodesForSpillArray() const2648 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
2649 return StoreSpillOpcodesArray[getSpillTarget()];
2650 }
2651
getLoadOpcodesForSpillArray() const2652 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
2653 return LoadSpillOpcodesArray[getSpillTarget()];
2654 }
2655
fixupIsDeadOrKill(MachineInstr & StartMI,MachineInstr & EndMI,unsigned RegNo) const2656 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr &StartMI, MachineInstr &EndMI,
2657 unsigned RegNo) const {
2658
2659 // Instructions between [StartMI, EndMI] should be in same basic block.
2660 assert((StartMI.getParent() == EndMI.getParent()) &&
2661 "Instructions are not in same basic block");
2662
2663 bool IsKillSet = false;
2664
2665 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
2666 MachineOperand &MO = MI.getOperand(Index);
2667 if (MO.isReg() && MO.isUse() && MO.isKill() &&
2668 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
2669 MO.setIsKill(false);
2670 };
2671
2672 // Set killed flag for EndMI.
2673 // No need to do anything if EndMI defines RegNo.
2674 int UseIndex =
2675 EndMI.findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
2676 if (UseIndex != -1) {
2677 EndMI.getOperand(UseIndex).setIsKill(true);
2678 IsKillSet = true;
2679 // Clear killed flag for other EndMI operands related to RegNo. In some
2680 // upexpected cases, killed may be set multiple times for same register
2681 // operand in same MI.
2682 for (int i = 0, e = EndMI.getNumOperands(); i != e; ++i)
2683 if (i != UseIndex)
2684 clearOperandKillInfo(EndMI, i);
2685 }
2686
2687 // Walking the inst in reverse order (EndMI -> StartMI].
2688 MachineBasicBlock::reverse_iterator It = EndMI;
2689 MachineBasicBlock::reverse_iterator E = EndMI.getParent()->rend();
2690 // EndMI has been handled above, skip it here.
2691 It++;
2692 MachineOperand *MO = nullptr;
2693 for (; It != E; ++It) {
2694 // Skip insturctions which could not be a def/use of RegNo.
2695 if (It->isDebugInstr() || It->isPosition())
2696 continue;
2697
2698 // Clear killed flag for all It operands related to RegNo. In some
2699 // upexpected cases, killed may be set multiple times for same register
2700 // operand in same MI.
2701 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
2702 clearOperandKillInfo(*It, i);
2703
2704 // If killed is not set, set killed for its last use or set dead for its def
2705 // if no use found.
2706 if (!IsKillSet) {
2707 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
2708 // Use found, set it killed.
2709 IsKillSet = true;
2710 MO->setIsKill(true);
2711 continue;
2712 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
2713 &getRegisterInfo()))) {
2714 // No use found, set dead for its def.
2715 assert(&*It == &StartMI && "No new def between StartMI and EndMI.");
2716 MO->setIsDead(true);
2717 break;
2718 }
2719 }
2720
2721 if ((&*It) == &StartMI)
2722 break;
2723 }
2724 // Ensure RegMo liveness is killed after EndMI.
2725 assert((IsKillSet || (MO && MO->isDead())) &&
2726 "RegNo should be killed or dead");
2727 }
2728
2729 // This opt tries to convert the following imm form to an index form to save an
2730 // add for stack variables.
2731 // Return false if no such pattern found.
2732 //
2733 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2734 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
2735 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
2736 //
2737 // can be converted to:
2738 //
2739 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
2740 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
2741 //
2742 // In order to eliminate ADD instr, make sure that:
2743 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
2744 // new ADDI instr and ADDI can only take int16 Imm.
2745 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
2746 // between ADDI and ADD instr since its original def in ADDI will be changed
2747 // in new ADDI instr. And also there should be no new def for it between
2748 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
2749 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
2750 // between ADD and Imm instr since ADD instr will be eliminated.
2751 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
2752 // moved to Index instr.
foldFrameOffset(MachineInstr & MI) const2753 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
2754 MachineFunction *MF = MI.getParent()->getParent();
2755 MachineRegisterInfo *MRI = &MF->getRegInfo();
2756 bool PostRA = !MRI->isSSA();
2757 // Do this opt after PEI which is after RA. The reason is stack slot expansion
2758 // in PEI may expose such opportunities since in PEI, stack slot offsets to
2759 // frame base(OffsetAddi) are determined.
2760 if (!PostRA)
2761 return false;
2762 unsigned ToBeDeletedReg = 0;
2763 int64_t OffsetImm = 0;
2764 unsigned XFormOpcode = 0;
2765 ImmInstrInfo III;
2766
2767 // Check if Imm instr meets requirement.
2768 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
2769 III))
2770 return false;
2771
2772 bool OtherIntermediateUse = false;
2773 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
2774
2775 // Exit if there is other use between ADD and Imm instr or no def found.
2776 if (OtherIntermediateUse || !ADDMI)
2777 return false;
2778
2779 // Check if ADD instr meets requirement.
2780 if (!isADDInstrEligibleForFolding(*ADDMI))
2781 return false;
2782
2783 unsigned ScaleRegIdx = 0;
2784 int64_t OffsetAddi = 0;
2785 MachineInstr *ADDIMI = nullptr;
2786
2787 // Check if there is a valid ToBeChangedReg in ADDMI.
2788 // 1: It must be killed.
2789 // 2: Its definition must be a valid ADDIMI.
2790 // 3: It must satify int16 offset requirement.
2791 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
2792 ScaleRegIdx = 2;
2793 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
2794 ScaleRegIdx = 1;
2795 else
2796 return false;
2797
2798 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
2799 unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg();
2800 unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
2801 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
2802 MachineBasicBlock::iterator End) {
2803 for (auto It = ++Start; It != End; It++)
2804 if (It->modifiesRegister(Reg, &getRegisterInfo()))
2805 return true;
2806 return false;
2807 };
2808
2809 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
2810 // treated as special zero when ScaleReg is R0/X0 register.
2811 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
2812 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
2813 return false;
2814
2815 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
2816 // and Imm Instr.
2817 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
2818 return false;
2819
2820 // Now start to do the transformation.
2821 LLVM_DEBUG(dbgs() << "Replace instruction: "
2822 << "\n");
2823 LLVM_DEBUG(ADDIMI->dump());
2824 LLVM_DEBUG(ADDMI->dump());
2825 LLVM_DEBUG(MI.dump());
2826 LLVM_DEBUG(dbgs() << "with: "
2827 << "\n");
2828
2829 // Update ADDI instr.
2830 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
2831
2832 // Update Imm instr.
2833 MI.setDesc(get(XFormOpcode));
2834 MI.getOperand(III.ImmOpNo)
2835 .ChangeToRegister(ScaleReg, false, false,
2836 ADDMI->getOperand(ScaleRegIdx).isKill());
2837
2838 MI.getOperand(III.OpNoForForwarding)
2839 .ChangeToRegister(ToBeChangedReg, false, false, true);
2840
2841 // Eliminate ADD instr.
2842 ADDMI->eraseFromParent();
2843
2844 LLVM_DEBUG(ADDIMI->dump());
2845 LLVM_DEBUG(MI.dump());
2846
2847 return true;
2848 }
2849
isADDIInstrEligibleForFolding(MachineInstr & ADDIMI,int64_t & Imm) const2850 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
2851 int64_t &Imm) const {
2852 unsigned Opc = ADDIMI.getOpcode();
2853
2854 // Exit if the instruction is not ADDI.
2855 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
2856 return false;
2857
2858 // The operand may not necessarily be an immediate - it could be a relocation.
2859 if (!ADDIMI.getOperand(2).isImm())
2860 return false;
2861
2862 Imm = ADDIMI.getOperand(2).getImm();
2863
2864 return true;
2865 }
2866
isADDInstrEligibleForFolding(MachineInstr & ADDMI) const2867 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
2868 unsigned Opc = ADDMI.getOpcode();
2869
2870 // Exit if the instruction is not ADD.
2871 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
2872 }
2873
isImmInstrEligibleForFolding(MachineInstr & MI,unsigned & ToBeDeletedReg,unsigned & XFormOpcode,int64_t & OffsetImm,ImmInstrInfo & III) const2874 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
2875 unsigned &ToBeDeletedReg,
2876 unsigned &XFormOpcode,
2877 int64_t &OffsetImm,
2878 ImmInstrInfo &III) const {
2879 // Only handle load/store.
2880 if (!MI.mayLoadOrStore())
2881 return false;
2882
2883 unsigned Opc = MI.getOpcode();
2884
2885 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
2886
2887 // Exit if instruction has no index form.
2888 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
2889 return false;
2890
2891 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
2892 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
2893 III, true))
2894 return false;
2895
2896 if (!III.IsSummingOperands)
2897 return false;
2898
2899 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
2900 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
2901 // Only support imm operands, not relocation slots or others.
2902 if (!ImmOperand.isImm())
2903 return false;
2904
2905 assert(RegOperand.isReg() && "Instruction format is not right");
2906
2907 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
2908 if (!RegOperand.isKill())
2909 return false;
2910
2911 ToBeDeletedReg = RegOperand.getReg();
2912 OffsetImm = ImmOperand.getImm();
2913
2914 return true;
2915 }
2916
isValidToBeChangedReg(MachineInstr * ADDMI,unsigned Index,MachineInstr * & ADDIMI,int64_t & OffsetAddi,int64_t OffsetImm) const2917 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
2918 MachineInstr *&ADDIMI,
2919 int64_t &OffsetAddi,
2920 int64_t OffsetImm) const {
2921 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
2922 MachineOperand &MO = ADDMI->getOperand(Index);
2923
2924 if (!MO.isKill())
2925 return false;
2926
2927 bool OtherIntermediateUse = false;
2928
2929 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
2930 // Currently handle only one "add + Imminstr" pair case, exit if other
2931 // intermediate use for ToBeChangedReg found.
2932 // TODO: handle the cases where there are other "add + Imminstr" pairs
2933 // with same offset in Imminstr which is like:
2934 //
2935 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2936 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
2937 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
2938 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
2939 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
2940 //
2941 // can be converted to:
2942 //
2943 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
2944 // (OffsetAddi + OffsetImm)
2945 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
2946 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
2947
2948 if (OtherIntermediateUse || !ADDIMI)
2949 return false;
2950 // Check if ADDI instr meets requirement.
2951 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
2952 return false;
2953
2954 if (isInt<16>(OffsetAddi + OffsetImm))
2955 return true;
2956 return false;
2957 }
2958
2959 // If this instruction has an immediate form and one of its operands is a
2960 // result of a load-immediate or an add-immediate, convert it to
2961 // the immediate form if the constant is in range.
convertToImmediateForm(MachineInstr & MI,MachineInstr ** KilledDef) const2962 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
2963 MachineInstr **KilledDef) const {
2964 MachineFunction *MF = MI.getParent()->getParent();
2965 MachineRegisterInfo *MRI = &MF->getRegInfo();
2966 bool PostRA = !MRI->isSSA();
2967 bool SeenIntermediateUse = true;
2968 unsigned ForwardingOperand = ~0U;
2969 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
2970 SeenIntermediateUse);
2971 if (!DefMI)
2972 return false;
2973 assert(ForwardingOperand < MI.getNumOperands() &&
2974 "The forwarding operand needs to be valid at this point");
2975 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
2976 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
2977 if (KilledDef && KillFwdDefMI)
2978 *KilledDef = DefMI;
2979
2980 // If this is a imm instruction and its register operands is produced by ADDI,
2981 // put the imm into imm inst directly.
2982 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
2983 PPC::INSTRUCTION_LIST_END &&
2984 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
2985 return true;
2986
2987 ImmInstrInfo III;
2988 bool IsVFReg = MI.getOperand(0).isReg()
2989 ? isVFRegister(MI.getOperand(0).getReg())
2990 : false;
2991 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
2992 // If this is a reg+reg instruction that has a reg+imm form,
2993 // and one of the operands is produced by an add-immediate,
2994 // try to convert it.
2995 if (HasImmForm &&
2996 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
2997 KillFwdDefMI))
2998 return true;
2999
3000 // If this is a reg+reg instruction that has a reg+imm form,
3001 // and one of the operands is produced by LI, convert it now.
3002 if (HasImmForm &&
3003 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3004 return true;
3005
3006 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3007 // can be simpified to LI.
3008 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3009 return true;
3010
3011 return false;
3012 }
3013
instrHasImmForm(unsigned Opc,bool IsVFReg,ImmInstrInfo & III,bool PostRA) const3014 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3015 ImmInstrInfo &III, bool PostRA) const {
3016 // The vast majority of the instructions would need their operand 2 replaced
3017 // with an immediate when switching to the reg+imm form. A marked exception
3018 // are the update form loads/stores for which a constant operand 2 would need
3019 // to turn into a displacement and move operand 1 to the operand 2 position.
3020 III.ImmOpNo = 2;
3021 III.OpNoForForwarding = 2;
3022 III.ImmWidth = 16;
3023 III.ImmMustBeMultipleOf = 1;
3024 III.TruncateImmTo = 0;
3025 III.IsSummingOperands = false;
3026 switch (Opc) {
3027 default: return false;
3028 case PPC::ADD4:
3029 case PPC::ADD8:
3030 III.SignedImm = true;
3031 III.ZeroIsSpecialOrig = 0;
3032 III.ZeroIsSpecialNew = 1;
3033 III.IsCommutative = true;
3034 III.IsSummingOperands = true;
3035 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3036 break;
3037 case PPC::ADDC:
3038 case PPC::ADDC8:
3039 III.SignedImm = true;
3040 III.ZeroIsSpecialOrig = 0;
3041 III.ZeroIsSpecialNew = 0;
3042 III.IsCommutative = true;
3043 III.IsSummingOperands = true;
3044 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3045 break;
3046 case PPC::ADDC_rec:
3047 III.SignedImm = true;
3048 III.ZeroIsSpecialOrig = 0;
3049 III.ZeroIsSpecialNew = 0;
3050 III.IsCommutative = true;
3051 III.IsSummingOperands = true;
3052 III.ImmOpcode = PPC::ADDIC_rec;
3053 break;
3054 case PPC::SUBFC:
3055 case PPC::SUBFC8:
3056 III.SignedImm = true;
3057 III.ZeroIsSpecialOrig = 0;
3058 III.ZeroIsSpecialNew = 0;
3059 III.IsCommutative = false;
3060 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3061 break;
3062 case PPC::CMPW:
3063 case PPC::CMPD:
3064 III.SignedImm = true;
3065 III.ZeroIsSpecialOrig = 0;
3066 III.ZeroIsSpecialNew = 0;
3067 III.IsCommutative = false;
3068 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3069 break;
3070 case PPC::CMPLW:
3071 case PPC::CMPLD:
3072 III.SignedImm = false;
3073 III.ZeroIsSpecialOrig = 0;
3074 III.ZeroIsSpecialNew = 0;
3075 III.IsCommutative = false;
3076 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3077 break;
3078 case PPC::AND_rec:
3079 case PPC::AND8_rec:
3080 case PPC::OR:
3081 case PPC::OR8:
3082 case PPC::XOR:
3083 case PPC::XOR8:
3084 III.SignedImm = false;
3085 III.ZeroIsSpecialOrig = 0;
3086 III.ZeroIsSpecialNew = 0;
3087 III.IsCommutative = true;
3088 switch(Opc) {
3089 default: llvm_unreachable("Unknown opcode");
3090 case PPC::AND_rec:
3091 III.ImmOpcode = PPC::ANDI_rec;
3092 break;
3093 case PPC::AND8_rec:
3094 III.ImmOpcode = PPC::ANDI8_rec;
3095 break;
3096 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3097 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3098 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3099 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3100 }
3101 break;
3102 case PPC::RLWNM:
3103 case PPC::RLWNM8:
3104 case PPC::RLWNM_rec:
3105 case PPC::RLWNM8_rec:
3106 case PPC::SLW:
3107 case PPC::SLW8:
3108 case PPC::SLW_rec:
3109 case PPC::SLW8_rec:
3110 case PPC::SRW:
3111 case PPC::SRW8:
3112 case PPC::SRW_rec:
3113 case PPC::SRW8_rec:
3114 case PPC::SRAW:
3115 case PPC::SRAW_rec:
3116 III.SignedImm = false;
3117 III.ZeroIsSpecialOrig = 0;
3118 III.ZeroIsSpecialNew = 0;
3119 III.IsCommutative = false;
3120 // This isn't actually true, but the instructions ignore any of the
3121 // upper bits, so any immediate loaded with an LI is acceptable.
3122 // This does not apply to shift right algebraic because a value
3123 // out of range will produce a -1/0.
3124 III.ImmWidth = 16;
3125 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3126 Opc == PPC::RLWNM8_rec)
3127 III.TruncateImmTo = 5;
3128 else
3129 III.TruncateImmTo = 6;
3130 switch(Opc) {
3131 default: llvm_unreachable("Unknown opcode");
3132 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3133 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3134 case PPC::RLWNM_rec:
3135 III.ImmOpcode = PPC::RLWINM_rec;
3136 break;
3137 case PPC::RLWNM8_rec:
3138 III.ImmOpcode = PPC::RLWINM8_rec;
3139 break;
3140 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
3141 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
3142 case PPC::SLW_rec:
3143 III.ImmOpcode = PPC::RLWINM_rec;
3144 break;
3145 case PPC::SLW8_rec:
3146 III.ImmOpcode = PPC::RLWINM8_rec;
3147 break;
3148 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
3149 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
3150 case PPC::SRW_rec:
3151 III.ImmOpcode = PPC::RLWINM_rec;
3152 break;
3153 case PPC::SRW8_rec:
3154 III.ImmOpcode = PPC::RLWINM8_rec;
3155 break;
3156 case PPC::SRAW:
3157 III.ImmWidth = 5;
3158 III.TruncateImmTo = 0;
3159 III.ImmOpcode = PPC::SRAWI;
3160 break;
3161 case PPC::SRAW_rec:
3162 III.ImmWidth = 5;
3163 III.TruncateImmTo = 0;
3164 III.ImmOpcode = PPC::SRAWI_rec;
3165 break;
3166 }
3167 break;
3168 case PPC::RLDCL:
3169 case PPC::RLDCL_rec:
3170 case PPC::RLDCR:
3171 case PPC::RLDCR_rec:
3172 case PPC::SLD:
3173 case PPC::SLD_rec:
3174 case PPC::SRD:
3175 case PPC::SRD_rec:
3176 case PPC::SRAD:
3177 case PPC::SRAD_rec:
3178 III.SignedImm = false;
3179 III.ZeroIsSpecialOrig = 0;
3180 III.ZeroIsSpecialNew = 0;
3181 III.IsCommutative = false;
3182 // This isn't actually true, but the instructions ignore any of the
3183 // upper bits, so any immediate loaded with an LI is acceptable.
3184 // This does not apply to shift right algebraic because a value
3185 // out of range will produce a -1/0.
3186 III.ImmWidth = 16;
3187 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
3188 Opc == PPC::RLDCR_rec)
3189 III.TruncateImmTo = 6;
3190 else
3191 III.TruncateImmTo = 7;
3192 switch(Opc) {
3193 default: llvm_unreachable("Unknown opcode");
3194 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
3195 case PPC::RLDCL_rec:
3196 III.ImmOpcode = PPC::RLDICL_rec;
3197 break;
3198 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
3199 case PPC::RLDCR_rec:
3200 III.ImmOpcode = PPC::RLDICR_rec;
3201 break;
3202 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
3203 case PPC::SLD_rec:
3204 III.ImmOpcode = PPC::RLDICR_rec;
3205 break;
3206 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
3207 case PPC::SRD_rec:
3208 III.ImmOpcode = PPC::RLDICL_rec;
3209 break;
3210 case PPC::SRAD:
3211 III.ImmWidth = 6;
3212 III.TruncateImmTo = 0;
3213 III.ImmOpcode = PPC::SRADI;
3214 break;
3215 case PPC::SRAD_rec:
3216 III.ImmWidth = 6;
3217 III.TruncateImmTo = 0;
3218 III.ImmOpcode = PPC::SRADI_rec;
3219 break;
3220 }
3221 break;
3222 // Loads and stores:
3223 case PPC::LBZX:
3224 case PPC::LBZX8:
3225 case PPC::LHZX:
3226 case PPC::LHZX8:
3227 case PPC::LHAX:
3228 case PPC::LHAX8:
3229 case PPC::LWZX:
3230 case PPC::LWZX8:
3231 case PPC::LWAX:
3232 case PPC::LDX:
3233 case PPC::LFSX:
3234 case PPC::LFDX:
3235 case PPC::STBX:
3236 case PPC::STBX8:
3237 case PPC::STHX:
3238 case PPC::STHX8:
3239 case PPC::STWX:
3240 case PPC::STWX8:
3241 case PPC::STDX:
3242 case PPC::STFSX:
3243 case PPC::STFDX:
3244 III.SignedImm = true;
3245 III.ZeroIsSpecialOrig = 1;
3246 III.ZeroIsSpecialNew = 2;
3247 III.IsCommutative = true;
3248 III.IsSummingOperands = true;
3249 III.ImmOpNo = 1;
3250 III.OpNoForForwarding = 2;
3251 switch(Opc) {
3252 default: llvm_unreachable("Unknown opcode");
3253 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
3254 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
3255 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
3256 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
3257 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
3258 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
3259 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
3260 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
3261 case PPC::LWAX:
3262 III.ImmOpcode = PPC::LWA;
3263 III.ImmMustBeMultipleOf = 4;
3264 break;
3265 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
3266 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
3267 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
3268 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
3269 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
3270 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
3271 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
3272 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
3273 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
3274 case PPC::STDX:
3275 III.ImmOpcode = PPC::STD;
3276 III.ImmMustBeMultipleOf = 4;
3277 break;
3278 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
3279 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
3280 }
3281 break;
3282 case PPC::LBZUX:
3283 case PPC::LBZUX8:
3284 case PPC::LHZUX:
3285 case PPC::LHZUX8:
3286 case PPC::LHAUX:
3287 case PPC::LHAUX8:
3288 case PPC::LWZUX:
3289 case PPC::LWZUX8:
3290 case PPC::LDUX:
3291 case PPC::LFSUX:
3292 case PPC::LFDUX:
3293 case PPC::STBUX:
3294 case PPC::STBUX8:
3295 case PPC::STHUX:
3296 case PPC::STHUX8:
3297 case PPC::STWUX:
3298 case PPC::STWUX8:
3299 case PPC::STDUX:
3300 case PPC::STFSUX:
3301 case PPC::STFDUX:
3302 III.SignedImm = true;
3303 III.ZeroIsSpecialOrig = 2;
3304 III.ZeroIsSpecialNew = 3;
3305 III.IsCommutative = false;
3306 III.IsSummingOperands = true;
3307 III.ImmOpNo = 2;
3308 III.OpNoForForwarding = 3;
3309 switch(Opc) {
3310 default: llvm_unreachable("Unknown opcode");
3311 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
3312 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
3313 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
3314 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
3315 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
3316 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
3317 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
3318 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
3319 case PPC::LDUX:
3320 III.ImmOpcode = PPC::LDU;
3321 III.ImmMustBeMultipleOf = 4;
3322 break;
3323 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
3324 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
3325 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
3326 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
3327 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
3328 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
3329 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
3330 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
3331 case PPC::STDUX:
3332 III.ImmOpcode = PPC::STDU;
3333 III.ImmMustBeMultipleOf = 4;
3334 break;
3335 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
3336 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
3337 }
3338 break;
3339 // Power9 and up only. For some of these, the X-Form version has access to all
3340 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
3341 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
3342 // into or stored from is one of the VR registers.
3343 case PPC::LXVX:
3344 case PPC::LXSSPX:
3345 case PPC::LXSDX:
3346 case PPC::STXVX:
3347 case PPC::STXSSPX:
3348 case PPC::STXSDX:
3349 case PPC::XFLOADf32:
3350 case PPC::XFLOADf64:
3351 case PPC::XFSTOREf32:
3352 case PPC::XFSTOREf64:
3353 if (!Subtarget.hasP9Vector())
3354 return false;
3355 III.SignedImm = true;
3356 III.ZeroIsSpecialOrig = 1;
3357 III.ZeroIsSpecialNew = 2;
3358 III.IsCommutative = true;
3359 III.IsSummingOperands = true;
3360 III.ImmOpNo = 1;
3361 III.OpNoForForwarding = 2;
3362 III.ImmMustBeMultipleOf = 4;
3363 switch(Opc) {
3364 default: llvm_unreachable("Unknown opcode");
3365 case PPC::LXVX:
3366 III.ImmOpcode = PPC::LXV;
3367 III.ImmMustBeMultipleOf = 16;
3368 break;
3369 case PPC::LXSSPX:
3370 if (PostRA) {
3371 if (IsVFReg)
3372 III.ImmOpcode = PPC::LXSSP;
3373 else {
3374 III.ImmOpcode = PPC::LFS;
3375 III.ImmMustBeMultipleOf = 1;
3376 }
3377 break;
3378 }
3379 LLVM_FALLTHROUGH;
3380 case PPC::XFLOADf32:
3381 III.ImmOpcode = PPC::DFLOADf32;
3382 break;
3383 case PPC::LXSDX:
3384 if (PostRA) {
3385 if (IsVFReg)
3386 III.ImmOpcode = PPC::LXSD;
3387 else {
3388 III.ImmOpcode = PPC::LFD;
3389 III.ImmMustBeMultipleOf = 1;
3390 }
3391 break;
3392 }
3393 LLVM_FALLTHROUGH;
3394 case PPC::XFLOADf64:
3395 III.ImmOpcode = PPC::DFLOADf64;
3396 break;
3397 case PPC::STXVX:
3398 III.ImmOpcode = PPC::STXV;
3399 III.ImmMustBeMultipleOf = 16;
3400 break;
3401 case PPC::STXSSPX:
3402 if (PostRA) {
3403 if (IsVFReg)
3404 III.ImmOpcode = PPC::STXSSP;
3405 else {
3406 III.ImmOpcode = PPC::STFS;
3407 III.ImmMustBeMultipleOf = 1;
3408 }
3409 break;
3410 }
3411 LLVM_FALLTHROUGH;
3412 case PPC::XFSTOREf32:
3413 III.ImmOpcode = PPC::DFSTOREf32;
3414 break;
3415 case PPC::STXSDX:
3416 if (PostRA) {
3417 if (IsVFReg)
3418 III.ImmOpcode = PPC::STXSD;
3419 else {
3420 III.ImmOpcode = PPC::STFD;
3421 III.ImmMustBeMultipleOf = 1;
3422 }
3423 break;
3424 }
3425 LLVM_FALLTHROUGH;
3426 case PPC::XFSTOREf64:
3427 III.ImmOpcode = PPC::DFSTOREf64;
3428 break;
3429 }
3430 break;
3431 }
3432 return true;
3433 }
3434
3435 // Utility function for swaping two arbitrary operands of an instruction.
swapMIOperands(MachineInstr & MI,unsigned Op1,unsigned Op2)3436 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
3437 assert(Op1 != Op2 && "Cannot swap operand with itself.");
3438
3439 unsigned MaxOp = std::max(Op1, Op2);
3440 unsigned MinOp = std::min(Op1, Op2);
3441 MachineOperand MOp1 = MI.getOperand(MinOp);
3442 MachineOperand MOp2 = MI.getOperand(MaxOp);
3443 MI.RemoveOperand(std::max(Op1, Op2));
3444 MI.RemoveOperand(std::min(Op1, Op2));
3445
3446 // If the operands we are swapping are the two at the end (the common case)
3447 // we can just remove both and add them in the opposite order.
3448 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
3449 MI.addOperand(MOp2);
3450 MI.addOperand(MOp1);
3451 } else {
3452 // Store all operands in a temporary vector, remove them and re-add in the
3453 // right order.
3454 SmallVector<MachineOperand, 2> MOps;
3455 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
3456 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
3457 MOps.push_back(MI.getOperand(i));
3458 MI.RemoveOperand(i);
3459 }
3460 // MOp2 needs to be added next.
3461 MI.addOperand(MOp2);
3462 // Now add the rest.
3463 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
3464 if (i == MaxOp)
3465 MI.addOperand(MOp1);
3466 else {
3467 MI.addOperand(MOps.back());
3468 MOps.pop_back();
3469 }
3470 }
3471 }
3472 }
3473
3474 // Check if the 'MI' that has the index OpNoForForwarding
3475 // meets the requirement described in the ImmInstrInfo.
isUseMIElgibleForForwarding(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding) const3476 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
3477 const ImmInstrInfo &III,
3478 unsigned OpNoForForwarding
3479 ) const {
3480 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
3481 // would not work pre-RA, we can only do the check post RA.
3482 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3483 if (MRI.isSSA())
3484 return false;
3485
3486 // Cannot do the transform if MI isn't summing the operands.
3487 if (!III.IsSummingOperands)
3488 return false;
3489
3490 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
3491 if (!III.ZeroIsSpecialOrig)
3492 return false;
3493
3494 // We cannot do the transform if the operand we are trying to replace
3495 // isn't the same as the operand the instruction allows.
3496 if (OpNoForForwarding != III.OpNoForForwarding)
3497 return false;
3498
3499 // Check if the instruction we are trying to transform really has
3500 // the special zero register as its operand.
3501 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
3502 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
3503 return false;
3504
3505 // This machine instruction is convertible if it is,
3506 // 1. summing the operands.
3507 // 2. one of the operands is special zero register.
3508 // 3. the operand we are trying to replace is allowed by the MI.
3509 return true;
3510 }
3511
3512 // Check if the DefMI is the add inst and set the ImmMO and RegMO
3513 // accordingly.
isDefMIElgibleForForwarding(MachineInstr & DefMI,const ImmInstrInfo & III,MachineOperand * & ImmMO,MachineOperand * & RegMO) const3514 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
3515 const ImmInstrInfo &III,
3516 MachineOperand *&ImmMO,
3517 MachineOperand *&RegMO) const {
3518 unsigned Opc = DefMI.getOpcode();
3519 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
3520 return false;
3521
3522 assert(DefMI.getNumOperands() >= 3 &&
3523 "Add inst must have at least three operands");
3524 RegMO = &DefMI.getOperand(1);
3525 ImmMO = &DefMI.getOperand(2);
3526
3527 // Before RA, ADDI first operand could be a frame index.
3528 if (!RegMO->isReg())
3529 return false;
3530
3531 // This DefMI is elgible for forwarding if it is:
3532 // 1. add inst
3533 // 2. one of the operands is Imm/CPI/Global.
3534 return isAnImmediateOperand(*ImmMO);
3535 }
3536
isRegElgibleForForwarding(const MachineOperand & RegMO,const MachineInstr & DefMI,const MachineInstr & MI,bool KillDefMI,bool & IsFwdFeederRegKilled) const3537 bool PPCInstrInfo::isRegElgibleForForwarding(
3538 const MachineOperand &RegMO, const MachineInstr &DefMI,
3539 const MachineInstr &MI, bool KillDefMI,
3540 bool &IsFwdFeederRegKilled) const {
3541 // x = addi y, imm
3542 // ...
3543 // z = lfdx 0, x -> z = lfd imm(y)
3544 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
3545 // of "y" between the DEF of "x" and "z".
3546 // The query is only valid post RA.
3547 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3548 if (MRI.isSSA())
3549 return false;
3550
3551 Register Reg = RegMO.getReg();
3552
3553 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
3554 MachineBasicBlock::const_reverse_iterator It = MI;
3555 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
3556 It++;
3557 for (; It != E; ++It) {
3558 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3559 return false;
3560 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3561 IsFwdFeederRegKilled = true;
3562 // Made it to DefMI without encountering a clobber.
3563 if ((&*It) == &DefMI)
3564 break;
3565 }
3566 assert((&*It) == &DefMI && "DefMI is missing");
3567
3568 // If DefMI also defines the register to be forwarded, we can only forward it
3569 // if DefMI is being erased.
3570 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
3571 return KillDefMI;
3572
3573 return true;
3574 }
3575
isImmElgibleForForwarding(const MachineOperand & ImmMO,const MachineInstr & DefMI,const ImmInstrInfo & III,int64_t & Imm,int64_t BaseImm) const3576 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
3577 const MachineInstr &DefMI,
3578 const ImmInstrInfo &III,
3579 int64_t &Imm,
3580 int64_t BaseImm) const {
3581 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
3582 if (DefMI.getOpcode() == PPC::ADDItocL) {
3583 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
3584 // However, we know that, it is 16-bit width, and has the alignment of 4.
3585 // Check if the instruction met the requirement.
3586 if (III.ImmMustBeMultipleOf > 4 ||
3587 III.TruncateImmTo || III.ImmWidth != 16)
3588 return false;
3589
3590 // Going from XForm to DForm loads means that the displacement needs to be
3591 // not just an immediate but also a multiple of 4, or 16 depending on the
3592 // load. A DForm load cannot be represented if it is a multiple of say 2.
3593 // XForm loads do not have this restriction.
3594 if (ImmMO.isGlobal()) {
3595 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
3596 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
3597 return false;
3598 }
3599
3600 return true;
3601 }
3602
3603 if (ImmMO.isImm()) {
3604 // It is Imm, we need to check if the Imm fit the range.
3605 // Sign-extend to 64-bits.
3606 // DefMI may be folded with another imm form instruction, the result Imm is
3607 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
3608 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
3609
3610 if (Imm % III.ImmMustBeMultipleOf)
3611 return false;
3612 if (III.TruncateImmTo)
3613 Imm &= ((1 << III.TruncateImmTo) - 1);
3614 if (III.SignedImm) {
3615 APInt ActualValue(64, Imm, true);
3616 if (!ActualValue.isSignedIntN(III.ImmWidth))
3617 return false;
3618 } else {
3619 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
3620 if ((uint64_t)Imm > UnsignedMax)
3621 return false;
3622 }
3623 }
3624 else
3625 return false;
3626
3627 // This ImmMO is forwarded if it meets the requriement describle
3628 // in ImmInstrInfo
3629 return true;
3630 }
3631
simplifyToLI(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding,MachineInstr ** KilledDef) const3632 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
3633 unsigned OpNoForForwarding,
3634 MachineInstr **KilledDef) const {
3635 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
3636 !DefMI.getOperand(1).isImm())
3637 return false;
3638
3639 MachineFunction *MF = MI.getParent()->getParent();
3640 MachineRegisterInfo *MRI = &MF->getRegInfo();
3641 bool PostRA = !MRI->isSSA();
3642
3643 int64_t Immediate = DefMI.getOperand(1).getImm();
3644 // Sign-extend to 64-bits.
3645 int64_t SExtImm = SignExtend64<16>(Immediate);
3646
3647 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
3648 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
3649
3650 bool ReplaceWithLI = false;
3651 bool Is64BitLI = false;
3652 int64_t NewImm = 0;
3653 bool SetCR = false;
3654 unsigned Opc = MI.getOpcode();
3655 switch (Opc) {
3656 default:
3657 return false;
3658
3659 // FIXME: Any branches conditional on such a comparison can be made
3660 // unconditional. At this time, this happens too infrequently to be worth
3661 // the implementation effort, but if that ever changes, we could convert
3662 // such a pattern here.
3663 case PPC::CMPWI:
3664 case PPC::CMPLWI:
3665 case PPC::CMPDI:
3666 case PPC::CMPLDI: {
3667 // Doing this post-RA would require dataflow analysis to reliably find uses
3668 // of the CR register set by the compare.
3669 // No need to fixup killed/dead flag since this transformation is only valid
3670 // before RA.
3671 if (PostRA)
3672 return false;
3673 // If a compare-immediate is fed by an immediate and is itself an input of
3674 // an ISEL (the most common case) into a COPY of the correct register.
3675 bool Changed = false;
3676 Register DefReg = MI.getOperand(0).getReg();
3677 int64_t Comparand = MI.getOperand(2).getImm();
3678 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
3679 ? (Comparand | 0xFFFFFFFFFFFF0000)
3680 : Comparand;
3681
3682 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
3683 unsigned UseOpc = CompareUseMI.getOpcode();
3684 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
3685 continue;
3686 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
3687 Register TrueReg = CompareUseMI.getOperand(1).getReg();
3688 Register FalseReg = CompareUseMI.getOperand(2).getReg();
3689 unsigned RegToCopy =
3690 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
3691 if (RegToCopy == PPC::NoRegister)
3692 continue;
3693 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
3694 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
3695 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
3696 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
3697 CompareUseMI.RemoveOperand(3);
3698 CompareUseMI.RemoveOperand(2);
3699 continue;
3700 }
3701 LLVM_DEBUG(
3702 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
3703 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
3704 LLVM_DEBUG(dbgs() << "Is converted to:\n");
3705 // Convert to copy and remove unneeded operands.
3706 CompareUseMI.setDesc(get(PPC::COPY));
3707 CompareUseMI.RemoveOperand(3);
3708 CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1);
3709 CmpIselsConverted++;
3710 Changed = true;
3711 LLVM_DEBUG(CompareUseMI.dump());
3712 }
3713 if (Changed)
3714 return true;
3715 // This may end up incremented multiple times since this function is called
3716 // during a fixed-point transformation, but it is only meant to indicate the
3717 // presence of this opportunity.
3718 MissedConvertibleImmediateInstrs++;
3719 return false;
3720 }
3721
3722 // Immediate forms - may simply be convertable to an LI.
3723 case PPC::ADDI:
3724 case PPC::ADDI8: {
3725 // Does the sum fit in a 16-bit signed field?
3726 int64_t Addend = MI.getOperand(2).getImm();
3727 if (isInt<16>(Addend + SExtImm)) {
3728 ReplaceWithLI = true;
3729 Is64BitLI = Opc == PPC::ADDI8;
3730 NewImm = Addend + SExtImm;
3731 break;
3732 }
3733 return false;
3734 }
3735 case PPC::RLDICL:
3736 case PPC::RLDICL_rec:
3737 case PPC::RLDICL_32:
3738 case PPC::RLDICL_32_64: {
3739 // Use APInt's rotate function.
3740 int64_t SH = MI.getOperand(2).getImm();
3741 int64_t MB = MI.getOperand(3).getImm();
3742 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
3743 SExtImm, true);
3744 InVal = InVal.rotl(SH);
3745 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
3746 InVal &= Mask;
3747 // Can't replace negative values with an LI as that will sign-extend
3748 // and not clear the left bits. If we're setting the CR bit, we will use
3749 // ANDI_rec which won't sign extend, so that's safe.
3750 if (isUInt<15>(InVal.getSExtValue()) ||
3751 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
3752 ReplaceWithLI = true;
3753 Is64BitLI = Opc != PPC::RLDICL_32;
3754 NewImm = InVal.getSExtValue();
3755 SetCR = Opc == PPC::RLDICL_rec;
3756 break;
3757 }
3758 return false;
3759 }
3760 case PPC::RLWINM:
3761 case PPC::RLWINM8:
3762 case PPC::RLWINM_rec:
3763 case PPC::RLWINM8_rec: {
3764 int64_t SH = MI.getOperand(2).getImm();
3765 int64_t MB = MI.getOperand(3).getImm();
3766 int64_t ME = MI.getOperand(4).getImm();
3767 APInt InVal(32, SExtImm, true);
3768 InVal = InVal.rotl(SH);
3769 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
3770 InVal &= Mask;
3771 // Can't replace negative values with an LI as that will sign-extend
3772 // and not clear the left bits. If we're setting the CR bit, we will use
3773 // ANDI_rec which won't sign extend, so that's safe.
3774 bool ValueFits = isUInt<15>(InVal.getSExtValue());
3775 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
3776 isUInt<16>(InVal.getSExtValue()));
3777 if (ValueFits) {
3778 ReplaceWithLI = true;
3779 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
3780 NewImm = InVal.getSExtValue();
3781 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
3782 break;
3783 }
3784 return false;
3785 }
3786 case PPC::ORI:
3787 case PPC::ORI8:
3788 case PPC::XORI:
3789 case PPC::XORI8: {
3790 int64_t LogicalImm = MI.getOperand(2).getImm();
3791 int64_t Result = 0;
3792 if (Opc == PPC::ORI || Opc == PPC::ORI8)
3793 Result = LogicalImm | SExtImm;
3794 else
3795 Result = LogicalImm ^ SExtImm;
3796 if (isInt<16>(Result)) {
3797 ReplaceWithLI = true;
3798 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
3799 NewImm = Result;
3800 break;
3801 }
3802 return false;
3803 }
3804 }
3805
3806 if (ReplaceWithLI) {
3807 // We need to be careful with CR-setting instructions we're replacing.
3808 if (SetCR) {
3809 // We don't know anything about uses when we're out of SSA, so only
3810 // replace if the new immediate will be reproduced.
3811 bool ImmChanged = (SExtImm & NewImm) != NewImm;
3812 if (PostRA && ImmChanged)
3813 return false;
3814
3815 if (!PostRA) {
3816 // If the defining load-immediate has no other uses, we can just replace
3817 // the immediate with the new immediate.
3818 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
3819 DefMI.getOperand(1).setImm(NewImm);
3820
3821 // If we're not using the GPR result of the CR-setting instruction, we
3822 // just need to and with zero/non-zero depending on the new immediate.
3823 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
3824 if (NewImm) {
3825 assert(Immediate && "Transformation converted zero to non-zero?");
3826 NewImm = Immediate;
3827 }
3828 } else if (ImmChanged)
3829 return false;
3830 }
3831 }
3832
3833 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3834 LLVM_DEBUG(MI.dump());
3835 LLVM_DEBUG(dbgs() << "Fed by:\n");
3836 LLVM_DEBUG(DefMI.dump());
3837 LoadImmediateInfo LII;
3838 LII.Imm = NewImm;
3839 LII.Is64Bit = Is64BitLI;
3840 LII.SetCR = SetCR;
3841 // If we're setting the CR, the original load-immediate must be kept (as an
3842 // operand to ANDI_rec/ANDI8_rec).
3843 if (KilledDef && SetCR)
3844 *KilledDef = nullptr;
3845 replaceInstrWithLI(MI, LII);
3846
3847 // Fixup killed/dead flag after transformation.
3848 // Pattern:
3849 // ForwardingOperandReg = LI imm1
3850 // y = op2 imm2, ForwardingOperandReg(killed)
3851 if (IsForwardingOperandKilled)
3852 fixupIsDeadOrKill(DefMI, MI, ForwardingOperandReg);
3853
3854 LLVM_DEBUG(dbgs() << "With:\n");
3855 LLVM_DEBUG(MI.dump());
3856 return true;
3857 }
3858 return false;
3859 }
3860
transformToNewImmFormFedByAdd(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding) const3861 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
3862 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
3863 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3864 bool PostRA = !MRI->isSSA();
3865 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
3866 // for post-ra.
3867 if (PostRA)
3868 return false;
3869
3870 // Only handle load/store.
3871 if (!MI.mayLoadOrStore())
3872 return false;
3873
3874 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
3875
3876 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
3877 "MI must have x-form opcode");
3878
3879 // get Imm Form info.
3880 ImmInstrInfo III;
3881 bool IsVFReg = MI.getOperand(0).isReg()
3882 ? isVFRegister(MI.getOperand(0).getReg())
3883 : false;
3884
3885 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
3886 return false;
3887
3888 if (!III.IsSummingOperands)
3889 return false;
3890
3891 if (OpNoForForwarding != III.OpNoForForwarding)
3892 return false;
3893
3894 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
3895 if (!ImmOperandMI.isImm())
3896 return false;
3897
3898 // Check DefMI.
3899 MachineOperand *ImmMO = nullptr;
3900 MachineOperand *RegMO = nullptr;
3901 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3902 return false;
3903 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3904
3905 // Check Imm.
3906 // Set ImmBase from imm instruction as base and get new Imm inside
3907 // isImmElgibleForForwarding.
3908 int64_t ImmBase = ImmOperandMI.getImm();
3909 int64_t Imm = 0;
3910 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
3911 return false;
3912
3913 // Get killed info in case fixup needed after transformation.
3914 unsigned ForwardKilledOperandReg = ~0U;
3915 if (MI.getOperand(III.OpNoForForwarding).isKill())
3916 ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
3917
3918 // Do the transform
3919 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3920 LLVM_DEBUG(MI.dump());
3921 LLVM_DEBUG(dbgs() << "Fed by:\n");
3922 LLVM_DEBUG(DefMI.dump());
3923
3924 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
3925 MI.getOperand(III.OpNoForForwarding).setIsKill(RegMO->isKill());
3926 MI.getOperand(III.ImmOpNo).setImm(Imm);
3927
3928 // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
3929 if (DefMI.getParent() == MI.getParent()) {
3930 // Check if reg is killed between MI and DefMI.
3931 auto IsKilledFor = [&](unsigned Reg) {
3932 MachineBasicBlock::const_reverse_iterator It = MI;
3933 MachineBasicBlock::const_reverse_iterator E = DefMI;
3934 It++;
3935 for (; It != E; ++It) {
3936 if (It->killsRegister(Reg))
3937 return true;
3938 }
3939 return false;
3940 };
3941
3942 // Update kill flag
3943 if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
3944 fixupIsDeadOrKill(DefMI, MI, RegMO->getReg());
3945 if (ForwardKilledOperandReg != ~0U)
3946 fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
3947 }
3948
3949 LLVM_DEBUG(dbgs() << "With:\n");
3950 LLVM_DEBUG(MI.dump());
3951 return true;
3952 }
3953
3954 // If an X-Form instruction is fed by an add-immediate and one of its operands
3955 // is the literal zero, attempt to forward the source of the add-immediate to
3956 // the corresponding D-Form instruction with the displacement coming from
3957 // the immediate being added.
transformToImmFormFedByAdd(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding,MachineInstr & DefMI,bool KillDefMI) const3958 bool PPCInstrInfo::transformToImmFormFedByAdd(
3959 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
3960 MachineInstr &DefMI, bool KillDefMI) const {
3961 // RegMO ImmMO
3962 // | |
3963 // x = addi reg, imm <----- DefMI
3964 // y = op 0 , x <----- MI
3965 // |
3966 // OpNoForForwarding
3967 // Check if the MI meet the requirement described in the III.
3968 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
3969 return false;
3970
3971 // Check if the DefMI meet the requirement
3972 // described in the III. If yes, set the ImmMO and RegMO accordingly.
3973 MachineOperand *ImmMO = nullptr;
3974 MachineOperand *RegMO = nullptr;
3975 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3976 return false;
3977 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3978
3979 // As we get the Imm operand now, we need to check if the ImmMO meet
3980 // the requirement described in the III. If yes set the Imm.
3981 int64_t Imm = 0;
3982 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
3983 return false;
3984
3985 bool IsFwdFeederRegKilled = false;
3986 // Check if the RegMO can be forwarded to MI.
3987 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
3988 IsFwdFeederRegKilled))
3989 return false;
3990
3991 // Get killed info in case fixup needed after transformation.
3992 unsigned ForwardKilledOperandReg = ~0U;
3993 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3994 bool PostRA = !MRI.isSSA();
3995 if (PostRA && MI.getOperand(OpNoForForwarding).isKill())
3996 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg();
3997
3998 // We know that, the MI and DefMI both meet the pattern, and
3999 // the Imm also meet the requirement with the new Imm-form.
4000 // It is safe to do the transformation now.
4001 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4002 LLVM_DEBUG(MI.dump());
4003 LLVM_DEBUG(dbgs() << "Fed by:\n");
4004 LLVM_DEBUG(DefMI.dump());
4005
4006 // Update the base reg first.
4007 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4008 false, false,
4009 RegMO->isKill());
4010
4011 // Then, update the imm.
4012 if (ImmMO->isImm()) {
4013 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4014 // directly.
4015 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4016 }
4017 else {
4018 // Otherwise, it is Constant Pool Index(CPI) or Global,
4019 // which is relocation in fact. We need to replace the special zero
4020 // register with ImmMO.
4021 // Before that, we need to fixup the target flags for imm.
4022 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4023 if (DefMI.getOpcode() == PPC::ADDItocL)
4024 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4025
4026 // MI didn't have the interface such as MI.setOperand(i) though
4027 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4028 // ImmMO, we need to remove ZERO operand and all the operands behind it,
4029 // and, add the ImmMO, then, move back all the operands behind ZERO.
4030 SmallVector<MachineOperand, 2> MOps;
4031 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4032 MOps.push_back(MI.getOperand(i));
4033 MI.RemoveOperand(i);
4034 }
4035
4036 // Remove the last MO in the list, which is ZERO operand in fact.
4037 MOps.pop_back();
4038 // Add the imm operand.
4039 MI.addOperand(*ImmMO);
4040 // Now add the rest back.
4041 for (auto &MO : MOps)
4042 MI.addOperand(MO);
4043 }
4044
4045 // Update the opcode.
4046 MI.setDesc(get(III.ImmOpcode));
4047
4048 // Fix up killed/dead flag after transformation.
4049 // Pattern 1:
4050 // x = ADD KilledFwdFeederReg, imm
4051 // n = opn KilledFwdFeederReg(killed), regn
4052 // y = XOP 0, x
4053 // Pattern 2:
4054 // x = ADD reg(killed), imm
4055 // y = XOP 0, x
4056 if (IsFwdFeederRegKilled || RegMO->isKill())
4057 fixupIsDeadOrKill(DefMI, MI, RegMO->getReg());
4058 // Pattern 3:
4059 // ForwardKilledOperandReg = ADD reg, imm
4060 // y = XOP 0, ForwardKilledOperandReg(killed)
4061 if (ForwardKilledOperandReg != ~0U)
4062 fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
4063
4064 LLVM_DEBUG(dbgs() << "With:\n");
4065 LLVM_DEBUG(MI.dump());
4066
4067 return true;
4068 }
4069
transformToImmFormFedByLI(MachineInstr & MI,const ImmInstrInfo & III,unsigned ConstantOpNo,MachineInstr & DefMI) const4070 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4071 const ImmInstrInfo &III,
4072 unsigned ConstantOpNo,
4073 MachineInstr &DefMI) const {
4074 // DefMI must be LI or LI8.
4075 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4076 !DefMI.getOperand(1).isImm())
4077 return false;
4078
4079 // Get Imm operand and Sign-extend to 64-bits.
4080 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4081
4082 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4083 bool PostRA = !MRI.isSSA();
4084 // Exit early if we can't convert this.
4085 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4086 return false;
4087 if (Imm % III.ImmMustBeMultipleOf)
4088 return false;
4089 if (III.TruncateImmTo)
4090 Imm &= ((1 << III.TruncateImmTo) - 1);
4091 if (III.SignedImm) {
4092 APInt ActualValue(64, Imm, true);
4093 if (!ActualValue.isSignedIntN(III.ImmWidth))
4094 return false;
4095 } else {
4096 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4097 if ((uint64_t)Imm > UnsignedMax)
4098 return false;
4099 }
4100
4101 // If we're post-RA, the instructions don't agree on whether register zero is
4102 // special, we can transform this as long as the register operand that will
4103 // end up in the location where zero is special isn't R0.
4104 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4105 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4106 III.ZeroIsSpecialNew + 1;
4107 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4108 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4109 // If R0 is in the operand where zero is special for the new instruction,
4110 // it is unsafe to transform if the constant operand isn't that operand.
4111 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4112 ConstantOpNo != III.ZeroIsSpecialNew)
4113 return false;
4114 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4115 ConstantOpNo != PosForOrigZero)
4116 return false;
4117 }
4118
4119 // Get killed info in case fixup needed after transformation.
4120 unsigned ForwardKilledOperandReg = ~0U;
4121 if (PostRA && MI.getOperand(ConstantOpNo).isKill())
4122 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg();
4123
4124 unsigned Opc = MI.getOpcode();
4125 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4126 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4127 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4128 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4129 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4130 Opc == PPC::SRD || Opc == PPC::SRD_rec;
4131 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4132 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4133 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4134 Opc == PPC::SRD_rec;
4135
4136 MI.setDesc(get(III.ImmOpcode));
4137 if (ConstantOpNo == III.OpNoForForwarding) {
4138 // Converting shifts to immediate form is a bit tricky since they may do
4139 // one of three things:
4140 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4141 // 2. If the shift amount is zero, the result is unchanged (save for maybe
4142 // setting CR0)
4143 // 3. If the shift amount is in [1, OpSize), it's just a shift
4144 if (SpecialShift32 || SpecialShift64) {
4145 LoadImmediateInfo LII;
4146 LII.Imm = 0;
4147 LII.SetCR = SetCR;
4148 LII.Is64Bit = SpecialShift64;
4149 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
4150 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
4151 replaceInstrWithLI(MI, LII);
4152 // Shifts by zero don't change the value. If we don't need to set CR0,
4153 // just convert this to a COPY. Can't do this post-RA since we've already
4154 // cleaned up the copies.
4155 else if (!SetCR && ShAmt == 0 && !PostRA) {
4156 MI.RemoveOperand(2);
4157 MI.setDesc(get(PPC::COPY));
4158 } else {
4159 // The 32 bit and 64 bit instructions are quite different.
4160 if (SpecialShift32) {
4161 // Left shifts use (N, 0, 31-N).
4162 // Right shifts use (32-N, N, 31) if 0 < N < 32.
4163 // use (0, 0, 31) if N == 0.
4164 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
4165 uint64_t MB = RightShift ? ShAmt : 0;
4166 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
4167 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4168 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
4169 .addImm(ME);
4170 } else {
4171 // Left shifts use (N, 63-N).
4172 // Right shifts use (64-N, N) if 0 < N < 64.
4173 // use (0, 0) if N == 0.
4174 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
4175 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
4176 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4177 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
4178 }
4179 }
4180 } else
4181 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4182 }
4183 // Convert commutative instructions (switch the operands and convert the
4184 // desired one to an immediate.
4185 else if (III.IsCommutative) {
4186 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4187 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
4188 } else
4189 llvm_unreachable("Should have exited early!");
4190
4191 // For instructions for which the constant register replaces a different
4192 // operand than where the immediate goes, we need to swap them.
4193 if (III.OpNoForForwarding != III.ImmOpNo)
4194 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
4195
4196 // If the special R0/X0 register index are different for original instruction
4197 // and new instruction, we need to fix up the register class in new
4198 // instruction.
4199 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4200 if (III.ZeroIsSpecialNew) {
4201 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
4202 // need to fix up register class.
4203 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4204 if (Register::isVirtualRegister(RegToModify)) {
4205 const TargetRegisterClass *NewRC =
4206 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
4207 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
4208 MRI.setRegClass(RegToModify, NewRC);
4209 }
4210 }
4211 }
4212
4213 // Fix up killed/dead flag after transformation.
4214 // Pattern:
4215 // ForwardKilledOperandReg = LI imm
4216 // y = XOP reg, ForwardKilledOperandReg(killed)
4217 if (ForwardKilledOperandReg != ~0U)
4218 fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
4219 return true;
4220 }
4221
4222 const TargetRegisterClass *
updatedRC(const TargetRegisterClass * RC) const4223 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
4224 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
4225 return &PPC::VSRCRegClass;
4226 return RC;
4227 }
4228
getRecordFormOpcode(unsigned Opcode)4229 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
4230 return PPC::getRecordFormOpcode(Opcode);
4231 }
4232
4233 // This function returns true if the machine instruction
4234 // always outputs a value by sign-extending a 32 bit value,
4235 // i.e. 0 to 31-th bits are same as 32-th bit.
isSignExtendingOp(const MachineInstr & MI)4236 static bool isSignExtendingOp(const MachineInstr &MI) {
4237 int Opcode = MI.getOpcode();
4238 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS ||
4239 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec ||
4240 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA ||
4241 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 ||
4242 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 ||
4243 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX ||
4244 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU ||
4245 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4246 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 ||
4247 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX ||
4248 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB ||
4249 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH ||
4250 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 ||
4251 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW ||
4252 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 ||
4253 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 ||
4254 Opcode == PPC::EXTSB8_32_64)
4255 return true;
4256
4257 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33)
4258 return true;
4259
4260 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4261 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
4262 MI.getOperand(3).getImm() > 0 &&
4263 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4264 return true;
4265
4266 return false;
4267 }
4268
4269 // This function returns true if the machine instruction
4270 // always outputs zeros in higher 32 bits.
isZeroExtendingOp(const MachineInstr & MI)4271 static bool isZeroExtendingOp(const MachineInstr &MI) {
4272 int Opcode = MI.getOpcode();
4273 // The 16-bit immediate is sign-extended in li/lis.
4274 // If the most significant bit is zero, all higher bits are zero.
4275 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
4276 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
4277 int64_t Imm = MI.getOperand(1).getImm();
4278 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
4279 return true;
4280 }
4281
4282 // We have some variations of rotate-and-mask instructions
4283 // that clear higher 32-bits.
4284 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
4285 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
4286 Opcode == PPC::RLDICL_32_64) &&
4287 MI.getOperand(3).getImm() >= 32)
4288 return true;
4289
4290 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
4291 MI.getOperand(3).getImm() >= 32 &&
4292 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm())
4293 return true;
4294
4295 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4296 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
4297 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
4298 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4299 return true;
4300
4301 // There are other instructions that clear higher 32-bits.
4302 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec ||
4303 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec ||
4304 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 ||
4305 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec ||
4306 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec ||
4307 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW ||
4308 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec ||
4309 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI ||
4310 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI ||
4311 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX ||
4312 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX ||
4313 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX ||
4314 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ ||
4315 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX ||
4316 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 ||
4317 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 ||
4318 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 ||
4319 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 ||
4320 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4321 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec ||
4322 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec ||
4323 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec ||
4324 Opcode == PPC::MFVSRWZ)
4325 return true;
4326
4327 return false;
4328 }
4329
4330 // This function returns true if the input MachineInstr is a TOC save
4331 // instruction.
isTOCSaveMI(const MachineInstr & MI) const4332 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
4333 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
4334 return false;
4335 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
4336 unsigned StackOffset = MI.getOperand(1).getImm();
4337 Register StackReg = MI.getOperand(2).getReg();
4338 if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset)
4339 return true;
4340
4341 return false;
4342 }
4343
4344 // We limit the max depth to track incoming values of PHIs or binary ops
4345 // (e.g. AND) to avoid excessive cost.
4346 const unsigned MAX_DEPTH = 1;
4347
4348 bool
isSignOrZeroExtended(const MachineInstr & MI,bool SignExt,const unsigned Depth) const4349 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
4350 const unsigned Depth) const {
4351 const MachineFunction *MF = MI.getParent()->getParent();
4352 const MachineRegisterInfo *MRI = &MF->getRegInfo();
4353
4354 // If we know this instruction returns sign- or zero-extended result,
4355 // return true.
4356 if (SignExt ? isSignExtendingOp(MI):
4357 isZeroExtendingOp(MI))
4358 return true;
4359
4360 switch (MI.getOpcode()) {
4361 case PPC::COPY: {
4362 Register SrcReg = MI.getOperand(1).getReg();
4363
4364 // In both ELFv1 and v2 ABI, method parameters and the return value
4365 // are sign- or zero-extended.
4366 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
4367 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
4368 // We check the ZExt/SExt flags for a method parameter.
4369 if (MI.getParent()->getBasicBlock() ==
4370 &MF->getFunction().getEntryBlock()) {
4371 Register VReg = MI.getOperand(0).getReg();
4372 if (MF->getRegInfo().isLiveIn(VReg))
4373 return SignExt ? FuncInfo->isLiveInSExt(VReg) :
4374 FuncInfo->isLiveInZExt(VReg);
4375 }
4376
4377 // For a method return value, we check the ZExt/SExt flags in attribute.
4378 // We assume the following code sequence for method call.
4379 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
4380 // BL8_NOP @func,...
4381 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
4382 // %5 = COPY %x3; G8RC:%5
4383 if (SrcReg == PPC::X3) {
4384 const MachineBasicBlock *MBB = MI.getParent();
4385 MachineBasicBlock::const_instr_iterator II =
4386 MachineBasicBlock::const_instr_iterator(&MI);
4387 if (II != MBB->instr_begin() &&
4388 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) {
4389 const MachineInstr &CallMI = *(--II);
4390 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) {
4391 const Function *CalleeFn =
4392 dyn_cast<Function>(CallMI.getOperand(0).getGlobal());
4393 if (!CalleeFn)
4394 return false;
4395 const IntegerType *IntTy =
4396 dyn_cast<IntegerType>(CalleeFn->getReturnType());
4397 const AttributeSet &Attrs =
4398 CalleeFn->getAttributes().getRetAttributes();
4399 if (IntTy && IntTy->getBitWidth() <= 32)
4400 return Attrs.hasAttribute(SignExt ? Attribute::SExt :
4401 Attribute::ZExt);
4402 }
4403 }
4404 }
4405 }
4406
4407 // If this is a copy from another register, we recursively check source.
4408 if (!Register::isVirtualRegister(SrcReg))
4409 return false;
4410 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4411 if (SrcMI != NULL)
4412 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4413
4414 return false;
4415 }
4416
4417 case PPC::ANDI_rec:
4418 case PPC::ANDIS_rec:
4419 case PPC::ORI:
4420 case PPC::ORIS:
4421 case PPC::XORI:
4422 case PPC::XORIS:
4423 case PPC::ANDI8_rec:
4424 case PPC::ANDIS8_rec:
4425 case PPC::ORI8:
4426 case PPC::ORIS8:
4427 case PPC::XORI8:
4428 case PPC::XORIS8: {
4429 // logical operation with 16-bit immediate does not change the upper bits.
4430 // So, we track the operand register as we do for register copy.
4431 Register SrcReg = MI.getOperand(1).getReg();
4432 if (!Register::isVirtualRegister(SrcReg))
4433 return false;
4434 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4435 if (SrcMI != NULL)
4436 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4437
4438 return false;
4439 }
4440
4441 // If all incoming values are sign-/zero-extended,
4442 // the output of OR, ISEL or PHI is also sign-/zero-extended.
4443 case PPC::OR:
4444 case PPC::OR8:
4445 case PPC::ISEL:
4446 case PPC::PHI: {
4447 if (Depth >= MAX_DEPTH)
4448 return false;
4449
4450 // The input registers for PHI are operand 1, 3, ...
4451 // The input registers for others are operand 1 and 2.
4452 unsigned E = 3, D = 1;
4453 if (MI.getOpcode() == PPC::PHI) {
4454 E = MI.getNumOperands();
4455 D = 2;
4456 }
4457
4458 for (unsigned I = 1; I != E; I += D) {
4459 if (MI.getOperand(I).isReg()) {
4460 Register SrcReg = MI.getOperand(I).getReg();
4461 if (!Register::isVirtualRegister(SrcReg))
4462 return false;
4463 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4464 if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1))
4465 return false;
4466 }
4467 else
4468 return false;
4469 }
4470 return true;
4471 }
4472
4473 // If at least one of the incoming values of an AND is zero extended
4474 // then the output is also zero-extended. If both of the incoming values
4475 // are sign-extended then the output is also sign extended.
4476 case PPC::AND:
4477 case PPC::AND8: {
4478 if (Depth >= MAX_DEPTH)
4479 return false;
4480
4481 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg());
4482
4483 Register SrcReg1 = MI.getOperand(1).getReg();
4484 Register SrcReg2 = MI.getOperand(2).getReg();
4485
4486 if (!Register::isVirtualRegister(SrcReg1) ||
4487 !Register::isVirtualRegister(SrcReg2))
4488 return false;
4489
4490 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1);
4491 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2);
4492 if (!MISrc1 || !MISrc2)
4493 return false;
4494
4495 if(SignExt)
4496 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) &&
4497 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4498 else
4499 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) ||
4500 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4501 }
4502
4503 default:
4504 break;
4505 }
4506 return false;
4507 }
4508
isBDNZ(unsigned Opcode) const4509 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
4510 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
4511 }
4512
4513 namespace {
4514 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
4515 MachineInstr *Loop, *EndLoop, *LoopCount;
4516 MachineFunction *MF;
4517 const TargetInstrInfo *TII;
4518 int64_t TripCount;
4519
4520 public:
PPCPipelinerLoopInfo(MachineInstr * Loop,MachineInstr * EndLoop,MachineInstr * LoopCount)4521 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
4522 MachineInstr *LoopCount)
4523 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
4524 MF(Loop->getParent()->getParent()),
4525 TII(MF->getSubtarget().getInstrInfo()) {
4526 // Inspect the Loop instruction up-front, as it may be deleted when we call
4527 // createTripCountGreaterCondition.
4528 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
4529 TripCount = LoopCount->getOperand(1).getImm();
4530 else
4531 TripCount = -1;
4532 }
4533
shouldIgnoreForPipelining(const MachineInstr * MI) const4534 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4535 // Only ignore the terminator.
4536 return MI == EndLoop;
4537 }
4538
4539 Optional<bool>
createTripCountGreaterCondition(int TC,MachineBasicBlock & MBB,SmallVectorImpl<MachineOperand> & Cond)4540 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
4541 SmallVectorImpl<MachineOperand> &Cond) override {
4542 if (TripCount == -1) {
4543 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4544 // so we don't need to generate any thing here.
4545 Cond.push_back(MachineOperand::CreateImm(0));
4546 Cond.push_back(MachineOperand::CreateReg(
4547 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
4548 true));
4549 return {};
4550 }
4551
4552 return TripCount > TC;
4553 }
4554
setPreheader(MachineBasicBlock * NewPreheader)4555 void setPreheader(MachineBasicBlock *NewPreheader) override {
4556 // Do nothing. We want the LOOP setup instruction to stay in the *old*
4557 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
4558 }
4559
adjustTripCount(int TripCountAdjust)4560 void adjustTripCount(int TripCountAdjust) override {
4561 // If the loop trip count is a compile-time value, then just change the
4562 // value.
4563 if (LoopCount->getOpcode() == PPC::LI8 ||
4564 LoopCount->getOpcode() == PPC::LI) {
4565 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
4566 LoopCount->getOperand(1).setImm(TripCount);
4567 return;
4568 }
4569
4570 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4571 // so we don't need to generate any thing here.
4572 }
4573
disposed()4574 void disposed() override {
4575 Loop->eraseFromParent();
4576 // Ensure the loop setup instruction is deleted too.
4577 LoopCount->eraseFromParent();
4578 }
4579 };
4580 } // namespace
4581
4582 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
analyzeLoopForPipelining(MachineBasicBlock * LoopBB) const4583 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
4584 // We really "analyze" only hardware loops right now.
4585 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
4586 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
4587 if (Preheader == LoopBB)
4588 Preheader = *std::next(LoopBB->pred_begin());
4589 MachineFunction *MF = Preheader->getParent();
4590
4591 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
4592 SmallPtrSet<MachineBasicBlock *, 8> Visited;
4593 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
4594 Register LoopCountReg = LoopInst->getOperand(0).getReg();
4595 MachineRegisterInfo &MRI = MF->getRegInfo();
4596 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
4597 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
4598 }
4599 }
4600 return nullptr;
4601 }
4602
findLoopInstr(MachineBasicBlock & PreHeader,SmallPtrSet<MachineBasicBlock *,8> & Visited) const4603 MachineInstr *PPCInstrInfo::findLoopInstr(
4604 MachineBasicBlock &PreHeader,
4605 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
4606
4607 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
4608
4609 // The loop set-up instruction should be in preheader
4610 for (auto &I : PreHeader.instrs())
4611 if (I.getOpcode() == LOOPi)
4612 return &I;
4613 return nullptr;
4614 }
4615
4616 // Return true if get the base operand, byte offset of an instruction and the
4617 // memory width. Width is the size of memory that is being loaded/stored.
getMemOperandWithOffsetWidth(const MachineInstr & LdSt,const MachineOperand * & BaseReg,int64_t & Offset,unsigned & Width,const TargetRegisterInfo * TRI) const4618 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
4619 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
4620 unsigned &Width, const TargetRegisterInfo *TRI) const {
4621 if (!LdSt.mayLoadOrStore())
4622 return false;
4623
4624 // Handle only loads/stores with base register followed by immediate offset.
4625 if (LdSt.getNumExplicitOperands() != 3)
4626 return false;
4627 if (!LdSt.getOperand(1).isImm() || !LdSt.getOperand(2).isReg())
4628 return false;
4629
4630 if (!LdSt.hasOneMemOperand())
4631 return false;
4632
4633 Width = (*LdSt.memoperands_begin())->getSize();
4634 Offset = LdSt.getOperand(1).getImm();
4635 BaseReg = &LdSt.getOperand(2);
4636 return true;
4637 }
4638
areMemAccessesTriviallyDisjoint(const MachineInstr & MIa,const MachineInstr & MIb) const4639 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
4640 const MachineInstr &MIa, const MachineInstr &MIb) const {
4641 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
4642 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
4643
4644 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
4645 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4646 return false;
4647
4648 // Retrieve the base register, offset from the base register and width. Width
4649 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
4650 // base registers are identical, and the offset of a lower memory access +
4651 // the width doesn't overlap the offset of a higher memory access,
4652 // then the memory accesses are different.
4653 const TargetRegisterInfo *TRI = &getRegisterInfo();
4654 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
4655 int64_t OffsetA = 0, OffsetB = 0;
4656 unsigned int WidthA = 0, WidthB = 0;
4657 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
4658 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
4659 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
4660 int LowOffset = std::min(OffsetA, OffsetB);
4661 int HighOffset = std::max(OffsetA, OffsetB);
4662 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4663 if (LowOffset + LowWidth <= HighOffset)
4664 return true;
4665 }
4666 }
4667 return false;
4668 }
4669