1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/MachineConstantPool.h"
25 #include "llvm/CodeGen/MachineFrameInfo.h"
26 #include "llvm/CodeGen/MachineFunctionPass.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineMemOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/PseudoSourceValue.h"
31 #include "llvm/CodeGen/RegisterClassInfo.h"
32 #include "llvm/CodeGen/RegisterPressure.h"
33 #include "llvm/CodeGen/ScheduleDAG.h"
34 #include "llvm/CodeGen/SlotIndexes.h"
35 #include "llvm/CodeGen/StackMaps.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/TargetRegistry.h"
39 #include "llvm/Support/CommandLine.h"
40 #include "llvm/Support/Debug.h"
41 #include "llvm/Support/ErrorHandling.h"
42 #include "llvm/Support/raw_ostream.h"
43
44 using namespace llvm;
45
46 #define DEBUG_TYPE "ppc-instr-info"
47
48 #define GET_INSTRMAP_INFO
49 #define GET_INSTRINFO_CTOR_DTOR
50 #include "PPCGenInstrInfo.inc"
51
52 STATISTIC(NumStoreSPILLVSRRCAsVec,
53 "Number of spillvsrrc spilled to stack as vec");
54 STATISTIC(NumStoreSPILLVSRRCAsGpr,
55 "Number of spillvsrrc spilled to stack as gpr");
56 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
57 STATISTIC(CmpIselsConverted,
58 "Number of ISELs that depend on comparison of constants converted");
59 STATISTIC(MissedConvertibleImmediateInstrs,
60 "Number of compare-immediate instructions fed by constants");
61 STATISTIC(NumRcRotatesConvertedToRcAnd,
62 "Number of record-form rotates converted to record-form andi");
63
64 static cl::
65 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
66 cl::desc("Disable analysis for CTR loops"));
67
68 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
69 cl::desc("Disable compare instruction optimization"), cl::Hidden);
70
71 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
72 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
73 cl::Hidden);
74
75 static cl::opt<bool>
76 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
77 cl::desc("Use the old (incorrect) instruction latency calculation"));
78
79 static cl::opt<float>
80 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
81 cl::desc("register pressure factor for the transformations."));
82
83 static cl::opt<bool> EnableFMARegPressureReduction(
84 "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
85 cl::desc("enable register pressure reduce in machine combiner pass."));
86
87 // Pin the vtable to this file.
anchor()88 void PPCInstrInfo::anchor() {}
89
PPCInstrInfo(PPCSubtarget & STI)90 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
91 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
92 /* CatchRetOpcode */ -1,
93 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
94 Subtarget(STI), RI(STI.getTargetMachine()) {}
95
96 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
97 /// this target when scheduling the DAG.
98 ScheduleHazardRecognizer *
CreateTargetHazardRecognizer(const TargetSubtargetInfo * STI,const ScheduleDAG * DAG) const99 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
100 const ScheduleDAG *DAG) const {
101 unsigned Directive =
102 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
103 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
104 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
105 const InstrItineraryData *II =
106 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
107 return new ScoreboardHazardRecognizer(II, DAG);
108 }
109
110 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
111 }
112
113 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
114 /// to use for this target when scheduling the DAG.
115 ScheduleHazardRecognizer *
CreateTargetPostRAHazardRecognizer(const InstrItineraryData * II,const ScheduleDAG * DAG) const116 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
117 const ScheduleDAG *DAG) const {
118 unsigned Directive =
119 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
120
121 // FIXME: Leaving this as-is until we have POWER9 scheduling info
122 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
123 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
124
125 // Most subtargets use a PPC970 recognizer.
126 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
127 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
128 assert(DAG->TII && "No InstrInfo?");
129
130 return new PPCHazardRecognizer970(*DAG);
131 }
132
133 return new ScoreboardHazardRecognizer(II, DAG);
134 }
135
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr & MI,unsigned * PredCost) const136 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
137 const MachineInstr &MI,
138 unsigned *PredCost) const {
139 if (!ItinData || UseOldLatencyCalc)
140 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
141
142 // The default implementation of getInstrLatency calls getStageLatency, but
143 // getStageLatency does not do the right thing for us. While we have
144 // itinerary, most cores are fully pipelined, and so the itineraries only
145 // express the first part of the pipeline, not every stage. Instead, we need
146 // to use the listed output operand cycle number (using operand 0 here, which
147 // is an output).
148
149 unsigned Latency = 1;
150 unsigned DefClass = MI.getDesc().getSchedClass();
151 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
152 const MachineOperand &MO = MI.getOperand(i);
153 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
154 continue;
155
156 int Cycle = ItinData->getOperandCycle(DefClass, i);
157 if (Cycle < 0)
158 continue;
159
160 Latency = std::max(Latency, (unsigned) Cycle);
161 }
162
163 return Latency;
164 }
165
getOperandLatency(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const166 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
167 const MachineInstr &DefMI, unsigned DefIdx,
168 const MachineInstr &UseMI,
169 unsigned UseIdx) const {
170 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
171 UseMI, UseIdx);
172
173 if (!DefMI.getParent())
174 return Latency;
175
176 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
177 Register Reg = DefMO.getReg();
178
179 bool IsRegCR;
180 if (Register::isVirtualRegister(Reg)) {
181 const MachineRegisterInfo *MRI =
182 &DefMI.getParent()->getParent()->getRegInfo();
183 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
184 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
185 } else {
186 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
187 PPC::CRBITRCRegClass.contains(Reg);
188 }
189
190 if (UseMI.isBranch() && IsRegCR) {
191 if (Latency < 0)
192 Latency = getInstrLatency(ItinData, DefMI);
193
194 // On some cores, there is an additional delay between writing to a condition
195 // register, and using it from a branch.
196 unsigned Directive = Subtarget.getCPUDirective();
197 switch (Directive) {
198 default: break;
199 case PPC::DIR_7400:
200 case PPC::DIR_750:
201 case PPC::DIR_970:
202 case PPC::DIR_E5500:
203 case PPC::DIR_PWR4:
204 case PPC::DIR_PWR5:
205 case PPC::DIR_PWR5X:
206 case PPC::DIR_PWR6:
207 case PPC::DIR_PWR6X:
208 case PPC::DIR_PWR7:
209 case PPC::DIR_PWR8:
210 // FIXME: Is this needed for POWER9?
211 Latency += 2;
212 break;
213 }
214 }
215
216 return Latency;
217 }
218
219 /// This is an architecture-specific helper function of reassociateOps.
220 /// Set special operand attributes for new instructions after reassociation.
setSpecialOperandAttr(MachineInstr & OldMI1,MachineInstr & OldMI2,MachineInstr & NewMI1,MachineInstr & NewMI2) const221 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
222 MachineInstr &OldMI2,
223 MachineInstr &NewMI1,
224 MachineInstr &NewMI2) const {
225 // Propagate FP flags from the original instructions.
226 // But clear poison-generating flags because those may not be valid now.
227 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
228 NewMI1.setFlags(IntersectedFlags);
229 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
230 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
231 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
232
233 NewMI2.setFlags(IntersectedFlags);
234 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
235 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
236 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
237 }
238
setSpecialOperandAttr(MachineInstr & MI,uint16_t Flags) const239 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
240 uint16_t Flags) const {
241 MI.setFlags(Flags);
242 MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
243 MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
244 MI.clearFlag(MachineInstr::MIFlag::IsExact);
245 }
246
247 // This function does not list all associative and commutative operations, but
248 // only those worth feeding through the machine combiner in an attempt to
249 // reduce the critical path. Mostly, this means floating-point operations,
250 // because they have high latencies(>=5) (compared to other operations, such as
251 // and/or, which are also associative and commutative, but have low latencies).
isAssociativeAndCommutative(const MachineInstr & Inst) const252 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
253 switch (Inst.getOpcode()) {
254 // Floating point:
255 // FP Add:
256 case PPC::FADD:
257 case PPC::FADDS:
258 // FP Multiply:
259 case PPC::FMUL:
260 case PPC::FMULS:
261 // Altivec Add:
262 case PPC::VADDFP:
263 // VSX Add:
264 case PPC::XSADDDP:
265 case PPC::XVADDDP:
266 case PPC::XVADDSP:
267 case PPC::XSADDSP:
268 // VSX Multiply:
269 case PPC::XSMULDP:
270 case PPC::XVMULDP:
271 case PPC::XVMULSP:
272 case PPC::XSMULSP:
273 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
274 Inst.getFlag(MachineInstr::MIFlag::FmNsz);
275 // Fixed point:
276 // Multiply:
277 case PPC::MULHD:
278 case PPC::MULLD:
279 case PPC::MULHW:
280 case PPC::MULLW:
281 return true;
282 default:
283 return false;
284 }
285 }
286
287 #define InfoArrayIdxFMAInst 0
288 #define InfoArrayIdxFAddInst 1
289 #define InfoArrayIdxFMULInst 2
290 #define InfoArrayIdxAddOpIdx 3
291 #define InfoArrayIdxMULOpIdx 4
292 #define InfoArrayIdxFSubInst 5
293 // Array keeps info for FMA instructions:
294 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
295 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
296 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
297 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
298 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
299 // second MUL operand index is plus 1;
300 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
301 static const uint16_t FMAOpIdxInfo[][6] = {
302 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
303 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
304 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
305 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
306 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
307 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
308 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
309
310 // Check if an opcode is a FMA instruction. If it is, return the index in array
311 // FMAOpIdxInfo. Otherwise, return -1.
getFMAOpIdxInfo(unsigned Opcode) const312 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
313 for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++)
314 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
315 return I;
316 return -1;
317 }
318
319 // On PowerPC target, we have two kinds of patterns related to FMA:
320 // 1: Improve ILP.
321 // Try to reassociate FMA chains like below:
322 //
323 // Pattern 1:
324 // A = FADD X, Y (Leaf)
325 // B = FMA A, M21, M22 (Prev)
326 // C = FMA B, M31, M32 (Root)
327 // -->
328 // A = FMA X, M21, M22
329 // B = FMA Y, M31, M32
330 // C = FADD A, B
331 //
332 // Pattern 2:
333 // A = FMA X, M11, M12 (Leaf)
334 // B = FMA A, M21, M22 (Prev)
335 // C = FMA B, M31, M32 (Root)
336 // -->
337 // A = FMUL M11, M12
338 // B = FMA X, M21, M22
339 // D = FMA A, M31, M32
340 // C = FADD B, D
341 //
342 // breaking the dependency between A and B, allowing FMA to be executed in
343 // parallel (or back-to-back in a pipeline) instead of depending on each other.
344 //
345 // 2: Reduce register pressure.
346 // Try to reassociate FMA with FSUB and a constant like below:
347 // C is a floating point const.
348 //
349 // Pattern 1:
350 // A = FSUB X, Y (Leaf)
351 // D = FMA B, C, A (Root)
352 // -->
353 // A = FMA B, Y, -C
354 // D = FMA A, X, C
355 //
356 // Pattern 2:
357 // A = FSUB X, Y (Leaf)
358 // D = FMA B, A, C (Root)
359 // -->
360 // A = FMA B, Y, -C
361 // D = FMA A, X, C
362 //
363 // Before the transformation, A must be assigned with different hardware
364 // register with D. After the transformation, A and D must be assigned with
365 // same hardware register due to TIE attribute of FMA instructions.
366 //
getFMAPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns,bool DoRegPressureReduce) const367 bool PPCInstrInfo::getFMAPatterns(
368 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
369 bool DoRegPressureReduce) const {
370 MachineBasicBlock *MBB = Root.getParent();
371 const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
372 const TargetRegisterInfo *TRI = &getRegisterInfo();
373
374 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
375 for (const auto &MO : Instr.explicit_operands())
376 if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
377 return false;
378 return true;
379 };
380
381 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
382 unsigned OpType) {
383 if (Instr.getOpcode() !=
384 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
385 return false;
386
387 // Instruction can be reassociated.
388 // fast math flags may prohibit reassociation.
389 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
390 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
391 return false;
392
393 // Instruction operands are virtual registers for reassociation.
394 if (!IsAllOpsVirtualReg(Instr))
395 return false;
396
397 // For register pressure reassociation, the FSub must have only one use as
398 // we want to delete the sub to save its def.
399 if (OpType == InfoArrayIdxFSubInst &&
400 !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
401 return false;
402
403 return true;
404 };
405
406 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
407 int16_t &MulOpIdx, bool IsLeaf) {
408 int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
409 if (Idx < 0)
410 return false;
411
412 // Instruction can be reassociated.
413 // fast math flags may prohibit reassociation.
414 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
415 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
416 return false;
417
418 // Instruction operands are virtual registers for reassociation.
419 if (!IsAllOpsVirtualReg(Instr))
420 return false;
421
422 MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
423 if (IsLeaf)
424 return true;
425
426 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
427
428 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
429 MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
430 // If 'add' operand's def is not in current block, don't do ILP related opt.
431 if (!MIAdd || MIAdd->getParent() != MBB)
432 return false;
433
434 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
435 // as this fma will be changed later.
436 return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
437 };
438
439 int16_t AddOpIdx = -1;
440 int16_t MulOpIdx = -1;
441
442 bool IsUsedOnceL = false;
443 bool IsUsedOnceR = false;
444 MachineInstr *MULInstrL = nullptr;
445 MachineInstr *MULInstrR = nullptr;
446
447 auto IsRPReductionCandidate = [&]() {
448 // Currently, we only support float and double.
449 // FIXME: add support for other types.
450 unsigned Opcode = Root.getOpcode();
451 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
452 return false;
453
454 // Root must be a valid FMA like instruction.
455 // Treat it as leaf as we don't care its add operand.
456 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
457 assert((MulOpIdx >= 0) && "mul operand index not right!");
458 Register MULRegL = TRI->lookThruSingleUseCopyChain(
459 Root.getOperand(MulOpIdx).getReg(), MRI);
460 Register MULRegR = TRI->lookThruSingleUseCopyChain(
461 Root.getOperand(MulOpIdx + 1).getReg(), MRI);
462 if (!MULRegL && !MULRegR)
463 return false;
464
465 if (MULRegL && !MULRegR) {
466 MULRegR =
467 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
468 IsUsedOnceL = true;
469 } else if (!MULRegL && MULRegR) {
470 MULRegL =
471 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
472 IsUsedOnceR = true;
473 } else {
474 IsUsedOnceL = true;
475 IsUsedOnceR = true;
476 }
477
478 if (!Register::isVirtualRegister(MULRegL) ||
479 !Register::isVirtualRegister(MULRegR))
480 return false;
481
482 MULInstrL = MRI->getVRegDef(MULRegL);
483 MULInstrR = MRI->getVRegDef(MULRegR);
484 return true;
485 }
486 return false;
487 };
488
489 // Register pressure fma reassociation patterns.
490 if (DoRegPressureReduce && IsRPReductionCandidate()) {
491 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
492 // Register pressure pattern 1
493 if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
494 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
495 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
496 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA);
497 return true;
498 }
499
500 // Register pressure pattern 2
501 if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
502 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
503 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
504 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC);
505 return true;
506 }
507 }
508
509 // ILP fma reassociation patterns.
510 // Root must be a valid FMA like instruction.
511 AddOpIdx = -1;
512 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
513 return false;
514
515 assert((AddOpIdx >= 0) && "add operand index not right!");
516
517 Register RegB = Root.getOperand(AddOpIdx).getReg();
518 MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
519
520 // Prev must be a valid FMA like instruction.
521 AddOpIdx = -1;
522 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
523 return false;
524
525 assert((AddOpIdx >= 0) && "add operand index not right!");
526
527 Register RegA = Prev->getOperand(AddOpIdx).getReg();
528 MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
529 AddOpIdx = -1;
530 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
531 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
532 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
533 return true;
534 }
535 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
536 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
537 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
538 return true;
539 }
540 return false;
541 }
542
finalizeInsInstrs(MachineInstr & Root,MachineCombinerPattern & P,SmallVectorImpl<MachineInstr * > & InsInstrs) const543 void PPCInstrInfo::finalizeInsInstrs(
544 MachineInstr &Root, MachineCombinerPattern &P,
545 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
546 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
547
548 MachineFunction *MF = Root.getMF();
549 MachineRegisterInfo *MRI = &MF->getRegInfo();
550 const TargetRegisterInfo *TRI = &getRegisterInfo();
551 MachineConstantPool *MCP = MF->getConstantPool();
552
553 int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
554 if (Idx < 0)
555 return;
556
557 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
558
559 // For now we only need to fix up placeholder for register pressure reduce
560 // patterns.
561 Register ConstReg = 0;
562 switch (P) {
563 case MachineCombinerPattern::REASSOC_XY_BCA:
564 ConstReg =
565 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
566 break;
567 case MachineCombinerPattern::REASSOC_XY_BAC:
568 ConstReg =
569 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
570 break;
571 default:
572 // Not register pressure reduce patterns.
573 return;
574 }
575
576 MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
577 // Get const value from const pool.
578 const Constant *C = getConstantFromConstantPool(ConstDefInstr);
579 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
580
581 // Get negative fp const.
582 APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
583 F1.changeSign();
584 Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
585 Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
586
587 // Put negative fp const into constant pool.
588 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
589
590 MachineOperand *Placeholder = nullptr;
591 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
592 for (auto *Inst : InsInstrs) {
593 for (MachineOperand &Operand : Inst->explicit_operands()) {
594 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
595 if (Operand.getReg() == PPC::ZERO8) {
596 Placeholder = &Operand;
597 break;
598 }
599 }
600 }
601
602 assert(Placeholder && "Placeholder does not exist!");
603
604 // Generate instructions to load the const fp from constant pool.
605 // We only support PPC64 and medium code model.
606 Register LoadNewConst =
607 generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
608
609 // Fill the placeholder with the new load from constant pool.
610 Placeholder->setReg(LoadNewConst);
611 }
612
shouldReduceRegisterPressure(MachineBasicBlock * MBB,RegisterClassInfo * RegClassInfo) const613 bool PPCInstrInfo::shouldReduceRegisterPressure(
614 MachineBasicBlock *MBB, RegisterClassInfo *RegClassInfo) const {
615
616 if (!EnableFMARegPressureReduction)
617 return false;
618
619 // Currently, we only enable register pressure reducing in machine combiner
620 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
621 // support.
622 //
623 // So we need following instructions to access a TOC entry:
624 //
625 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
626 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
627 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
628 //
629 // FIXME: add more supported targets, like Small and Large code model, PPC32,
630 // AIX.
631 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
632 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
633 return false;
634
635 const TargetRegisterInfo *TRI = &getRegisterInfo();
636 MachineFunction *MF = MBB->getParent();
637 MachineRegisterInfo *MRI = &MF->getRegInfo();
638
639 auto GetMBBPressure = [&](MachineBasicBlock *MBB) -> std::vector<unsigned> {
640 RegionPressure Pressure;
641 RegPressureTracker RPTracker(Pressure);
642
643 // Initialize the register pressure tracker.
644 RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
645 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
646
647 for (MachineBasicBlock::iterator MII = MBB->instr_end(),
648 MIE = MBB->instr_begin();
649 MII != MIE; --MII) {
650 MachineInstr &MI = *std::prev(MII);
651 if (MI.isDebugValue() || MI.isDebugLabel())
652 continue;
653 RegisterOperands RegOpers;
654 RegOpers.collect(MI, *TRI, *MRI, false, false);
655 RPTracker.recedeSkipDebugValues();
656 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
657 RPTracker.recede(RegOpers);
658 }
659
660 // Close the RPTracker to finalize live ins.
661 RPTracker.closeRegion();
662
663 return RPTracker.getPressure().MaxSetPressure;
664 };
665
666 // For now we only care about float and double type fma.
667 unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
668 *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
669
670 // Only reduce register pressure when pressure is high.
671 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
672 (float)VSSRCLimit * FMARPFactor;
673 }
674
isLoadFromConstantPool(MachineInstr * I) const675 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
676 // I has only one memory operand which is load from constant pool.
677 if (!I->hasOneMemOperand())
678 return false;
679
680 MachineMemOperand *Op = I->memoperands()[0];
681 return Op->isLoad() && Op->getPseudoValue() &&
682 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
683 }
684
generateLoadForNewConst(unsigned Idx,MachineInstr * MI,Type * Ty,SmallVectorImpl<MachineInstr * > & InsInstrs) const685 Register PPCInstrInfo::generateLoadForNewConst(
686 unsigned Idx, MachineInstr *MI, Type *Ty,
687 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
688 // Now we only support PPC64, Medium code model and P9 with vector.
689 // We have immutable pattern to access const pool. See function
690 // shouldReduceRegisterPressure.
691 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
692 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
693 "Target not supported!\n");
694
695 MachineFunction *MF = MI->getMF();
696 MachineRegisterInfo *MRI = &MF->getRegInfo();
697
698 // Generate ADDIStocHA8
699 Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
700 MachineInstrBuilder TOCOffset =
701 BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
702 .addReg(PPC::X2)
703 .addConstantPoolIndex(Idx);
704
705 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
706 "Only float and double are supported!");
707
708 unsigned LoadOpcode;
709 // Should be float type or double type.
710 if (Ty->isFloatTy())
711 LoadOpcode = PPC::DFLOADf32;
712 else
713 LoadOpcode = PPC::DFLOADf64;
714
715 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
716 Register VReg2 = MRI->createVirtualRegister(RC);
717 MachineMemOperand *MMO = MF->getMachineMemOperand(
718 MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad,
719 Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
720
721 // Generate Load from constant pool.
722 MachineInstrBuilder Load =
723 BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
724 .addConstantPoolIndex(Idx)
725 .addReg(VReg1, getKillRegState(true))
726 .addMemOperand(MMO);
727
728 Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
729
730 // Insert the toc load instructions into InsInstrs.
731 InsInstrs.insert(InsInstrs.begin(), Load);
732 InsInstrs.insert(InsInstrs.begin(), TOCOffset);
733 return VReg2;
734 }
735
736 // This function returns the const value in constant pool if the \p I is a load
737 // from constant pool.
738 const Constant *
getConstantFromConstantPool(MachineInstr * I) const739 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
740 MachineFunction *MF = I->getMF();
741 MachineRegisterInfo *MRI = &MF->getRegInfo();
742 MachineConstantPool *MCP = MF->getConstantPool();
743 assert(I->mayLoad() && "Should be a load instruction.\n");
744 for (auto MO : I->uses()) {
745 if (!MO.isReg())
746 continue;
747 Register Reg = MO.getReg();
748 if (Reg == 0 || !Register::isVirtualRegister(Reg))
749 continue;
750 // Find the toc address.
751 MachineInstr *DefMI = MRI->getVRegDef(Reg);
752 for (auto MO2 : DefMI->uses())
753 if (MO2.isCPI())
754 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
755 }
756 return nullptr;
757 }
758
getMachineCombinerPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns,bool DoRegPressureReduce) const759 bool PPCInstrInfo::getMachineCombinerPatterns(
760 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
761 bool DoRegPressureReduce) const {
762 // Using the machine combiner in this way is potentially expensive, so
763 // restrict to when aggressive optimizations are desired.
764 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
765 return false;
766
767 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
768 return true;
769
770 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
771 DoRegPressureReduce);
772 }
773
genAlternativeCodeSequence(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const774 void PPCInstrInfo::genAlternativeCodeSequence(
775 MachineInstr &Root, MachineCombinerPattern Pattern,
776 SmallVectorImpl<MachineInstr *> &InsInstrs,
777 SmallVectorImpl<MachineInstr *> &DelInstrs,
778 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
779 switch (Pattern) {
780 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
781 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
782 case MachineCombinerPattern::REASSOC_XY_BCA:
783 case MachineCombinerPattern::REASSOC_XY_BAC:
784 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
785 break;
786 default:
787 // Reassociate default patterns.
788 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
789 DelInstrs, InstrIdxForVirtReg);
790 break;
791 }
792 }
793
reassociateFMA(MachineInstr & Root,MachineCombinerPattern Pattern,SmallVectorImpl<MachineInstr * > & InsInstrs,SmallVectorImpl<MachineInstr * > & DelInstrs,DenseMap<unsigned,unsigned> & InstrIdxForVirtReg) const794 void PPCInstrInfo::reassociateFMA(
795 MachineInstr &Root, MachineCombinerPattern Pattern,
796 SmallVectorImpl<MachineInstr *> &InsInstrs,
797 SmallVectorImpl<MachineInstr *> &DelInstrs,
798 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
799 MachineFunction *MF = Root.getMF();
800 MachineRegisterInfo &MRI = MF->getRegInfo();
801 const TargetRegisterInfo *TRI = &getRegisterInfo();
802 MachineOperand &OpC = Root.getOperand(0);
803 Register RegC = OpC.getReg();
804 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
805 MRI.constrainRegClass(RegC, RC);
806
807 unsigned FmaOp = Root.getOpcode();
808 int16_t Idx = getFMAOpIdxInfo(FmaOp);
809 assert(Idx >= 0 && "Root must be a FMA instruction");
810
811 bool IsILPReassociate =
812 (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
813 (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
814
815 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
816 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
817
818 MachineInstr *Prev = nullptr;
819 MachineInstr *Leaf = nullptr;
820 switch (Pattern) {
821 default:
822 llvm_unreachable("not recognized pattern!");
823 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
824 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
825 Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
826 Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
827 break;
828 case MachineCombinerPattern::REASSOC_XY_BAC: {
829 Register MULReg =
830 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
831 Leaf = MRI.getVRegDef(MULReg);
832 break;
833 }
834 case MachineCombinerPattern::REASSOC_XY_BCA: {
835 Register MULReg = TRI->lookThruCopyLike(
836 Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
837 Leaf = MRI.getVRegDef(MULReg);
838 break;
839 }
840 }
841
842 uint16_t IntersectedFlags = 0;
843 if (IsILPReassociate)
844 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
845 else
846 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
847
848 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
849 bool &KillFlag) {
850 Reg = Operand.getReg();
851 MRI.constrainRegClass(Reg, RC);
852 KillFlag = Operand.isKill();
853 };
854
855 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
856 Register &MulOp2, Register &AddOp,
857 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
858 bool &AddOpKillFlag) {
859 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
860 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
861 GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
862 };
863
864 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
865 RegA21, RegB;
866 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
867 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
868 KillA11 = false, KillA21 = false, KillB = false;
869
870 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
871
872 if (IsILPReassociate)
873 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
874
875 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
876 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
877 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
878 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
879 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
880 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
881 } else {
882 // Get FSUB instruction info.
883 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
884 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
885 }
886
887 // Create new virtual registers for the new results instead of
888 // recycling legacy ones because the MachineCombiner's computation of the
889 // critical path requires a new register definition rather than an existing
890 // one.
891 // For register pressure reassociation, we only need create one virtual
892 // register for the new fma.
893 Register NewVRA = MRI.createVirtualRegister(RC);
894 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
895
896 Register NewVRB = 0;
897 if (IsILPReassociate) {
898 NewVRB = MRI.createVirtualRegister(RC);
899 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
900 }
901
902 Register NewVRD = 0;
903 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
904 NewVRD = MRI.createVirtualRegister(RC);
905 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
906 }
907
908 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
909 Register RegMul1, bool KillRegMul1,
910 Register RegMul2, bool KillRegMul2) {
911 MI->getOperand(AddOpIdx).setReg(RegAdd);
912 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
913 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
914 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
915 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
916 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
917 };
918
919 MachineInstrBuilder NewARegPressure, NewCRegPressure;
920 switch (Pattern) {
921 default:
922 llvm_unreachable("not recognized pattern!");
923 case MachineCombinerPattern::REASSOC_XY_AMM_BMM: {
924 // Create new instructions for insertion.
925 MachineInstrBuilder MINewB =
926 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
927 .addReg(RegX, getKillRegState(KillX))
928 .addReg(RegM21, getKillRegState(KillM21))
929 .addReg(RegM22, getKillRegState(KillM22));
930 MachineInstrBuilder MINewA =
931 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
932 .addReg(RegY, getKillRegState(KillY))
933 .addReg(RegM31, getKillRegState(KillM31))
934 .addReg(RegM32, getKillRegState(KillM32));
935 // If AddOpIdx is not 1, adjust the order.
936 if (AddOpIdx != 1) {
937 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
938 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
939 }
940
941 MachineInstrBuilder MINewC =
942 BuildMI(*MF, Root.getDebugLoc(),
943 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
944 .addReg(NewVRB, getKillRegState(true))
945 .addReg(NewVRA, getKillRegState(true));
946
947 // Update flags for newly created instructions.
948 setSpecialOperandAttr(*MINewA, IntersectedFlags);
949 setSpecialOperandAttr(*MINewB, IntersectedFlags);
950 setSpecialOperandAttr(*MINewC, IntersectedFlags);
951
952 // Record new instructions for insertion.
953 InsInstrs.push_back(MINewA);
954 InsInstrs.push_back(MINewB);
955 InsInstrs.push_back(MINewC);
956 break;
957 }
958 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
959 assert(NewVRD && "new FMA register not created!");
960 // Create new instructions for insertion.
961 MachineInstrBuilder MINewA =
962 BuildMI(*MF, Leaf->getDebugLoc(),
963 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
964 .addReg(RegM11, getKillRegState(KillM11))
965 .addReg(RegM12, getKillRegState(KillM12));
966 MachineInstrBuilder MINewB =
967 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
968 .addReg(RegX, getKillRegState(KillX))
969 .addReg(RegM21, getKillRegState(KillM21))
970 .addReg(RegM22, getKillRegState(KillM22));
971 MachineInstrBuilder MINewD =
972 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
973 .addReg(NewVRA, getKillRegState(true))
974 .addReg(RegM31, getKillRegState(KillM31))
975 .addReg(RegM32, getKillRegState(KillM32));
976 // If AddOpIdx is not 1, adjust the order.
977 if (AddOpIdx != 1) {
978 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
979 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
980 KillM32);
981 }
982
983 MachineInstrBuilder MINewC =
984 BuildMI(*MF, Root.getDebugLoc(),
985 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
986 .addReg(NewVRB, getKillRegState(true))
987 .addReg(NewVRD, getKillRegState(true));
988
989 // Update flags for newly created instructions.
990 setSpecialOperandAttr(*MINewA, IntersectedFlags);
991 setSpecialOperandAttr(*MINewB, IntersectedFlags);
992 setSpecialOperandAttr(*MINewD, IntersectedFlags);
993 setSpecialOperandAttr(*MINewC, IntersectedFlags);
994
995 // Record new instructions for insertion.
996 InsInstrs.push_back(MINewA);
997 InsInstrs.push_back(MINewB);
998 InsInstrs.push_back(MINewD);
999 InsInstrs.push_back(MINewC);
1000 break;
1001 }
1002 case MachineCombinerPattern::REASSOC_XY_BAC:
1003 case MachineCombinerPattern::REASSOC_XY_BCA: {
1004 Register VarReg;
1005 bool KillVarReg = false;
1006 if (Pattern == MachineCombinerPattern::REASSOC_XY_BCA) {
1007 VarReg = RegM31;
1008 KillVarReg = KillM31;
1009 } else {
1010 VarReg = RegM32;
1011 KillVarReg = KillM32;
1012 }
1013 // We don't want to get negative const from memory pool too early, as the
1014 // created entry will not be deleted even if it has no users. Since all
1015 // operand of Leaf and Root are virtual register, we use zero register
1016 // here as a placeholder. When the InsInstrs is selected in
1017 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1018 // with a virtual register which is a load from constant pool.
1019 NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1020 .addReg(RegB, getKillRegState(RegB))
1021 .addReg(RegY, getKillRegState(KillY))
1022 .addReg(PPC::ZERO8);
1023 NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1024 .addReg(NewVRA, getKillRegState(true))
1025 .addReg(RegX, getKillRegState(KillX))
1026 .addReg(VarReg, getKillRegState(KillVarReg));
1027 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1028 // both at index 1, no need to adjust.
1029 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1030 // the operand index here.
1031 break;
1032 }
1033 }
1034
1035 if (!IsILPReassociate) {
1036 setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1037 setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1038
1039 InsInstrs.push_back(NewARegPressure);
1040 InsInstrs.push_back(NewCRegPressure);
1041 }
1042
1043 assert(!InsInstrs.empty() &&
1044 "Insertion instructions set should not be empty!");
1045
1046 // Record old instructions for deletion.
1047 DelInstrs.push_back(Leaf);
1048 if (IsILPReassociate)
1049 DelInstrs.push_back(Prev);
1050 DelInstrs.push_back(&Root);
1051 }
1052
1053 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
isCoalescableExtInstr(const MachineInstr & MI,Register & SrcReg,Register & DstReg,unsigned & SubIdx) const1054 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1055 Register &SrcReg, Register &DstReg,
1056 unsigned &SubIdx) const {
1057 switch (MI.getOpcode()) {
1058 default: return false;
1059 case PPC::EXTSW:
1060 case PPC::EXTSW_32:
1061 case PPC::EXTSW_32_64:
1062 SrcReg = MI.getOperand(1).getReg();
1063 DstReg = MI.getOperand(0).getReg();
1064 SubIdx = PPC::sub_32;
1065 return true;
1066 }
1067 }
1068
isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const1069 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1070 int &FrameIndex) const {
1071 unsigned Opcode = MI.getOpcode();
1072 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1073 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
1074
1075 if (End != std::find(OpcodesForSpill, End, Opcode)) {
1076 // Check for the operands added by addFrameReference (the immediate is the
1077 // offset which defaults to 0).
1078 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1079 MI.getOperand(2).isFI()) {
1080 FrameIndex = MI.getOperand(2).getIndex();
1081 return MI.getOperand(0).getReg();
1082 }
1083 }
1084 return 0;
1085 }
1086
1087 // For opcodes with the ReMaterializable flag set, this function is called to
1088 // verify the instruction is really rematable.
isReallyTriviallyReMaterializable(const MachineInstr & MI,AliasAnalysis * AA) const1089 bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
1090 AliasAnalysis *AA) const {
1091 switch (MI.getOpcode()) {
1092 default:
1093 // This function should only be called for opcodes with the ReMaterializable
1094 // flag set.
1095 llvm_unreachable("Unknown rematerializable operation!");
1096 break;
1097 case PPC::LI:
1098 case PPC::LI8:
1099 case PPC::PLI:
1100 case PPC::PLI8:
1101 case PPC::LIS:
1102 case PPC::LIS8:
1103 case PPC::ADDIStocHA:
1104 case PPC::ADDIStocHA8:
1105 case PPC::ADDItocL:
1106 case PPC::LOAD_STACK_GUARD:
1107 case PPC::XXLXORz:
1108 case PPC::XXLXORspz:
1109 case PPC::XXLXORdpz:
1110 case PPC::XXLEQVOnes:
1111 case PPC::XXSPLTI32DX:
1112 case PPC::XXSPLTIW:
1113 case PPC::XXSPLTIDP:
1114 case PPC::V_SET0B:
1115 case PPC::V_SET0H:
1116 case PPC::V_SET0:
1117 case PPC::V_SETALLONESB:
1118 case PPC::V_SETALLONESH:
1119 case PPC::V_SETALLONES:
1120 case PPC::CRSET:
1121 case PPC::CRUNSET:
1122 case PPC::XXSETACCZ:
1123 return true;
1124 }
1125 return false;
1126 }
1127
isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const1128 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1129 int &FrameIndex) const {
1130 unsigned Opcode = MI.getOpcode();
1131 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1132 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
1133
1134 if (End != std::find(OpcodesForSpill, End, Opcode)) {
1135 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1136 MI.getOperand(2).isFI()) {
1137 FrameIndex = MI.getOperand(2).getIndex();
1138 return MI.getOperand(0).getReg();
1139 }
1140 }
1141 return 0;
1142 }
1143
commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const1144 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1145 unsigned OpIdx1,
1146 unsigned OpIdx2) const {
1147 MachineFunction &MF = *MI.getParent()->getParent();
1148
1149 // Normal instructions can be commuted the obvious way.
1150 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1151 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1152 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1153 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1154 // changing the relative order of the mask operands might change what happens
1155 // to the high-bits of the mask (and, thus, the result).
1156
1157 // Cannot commute if it has a non-zero rotate count.
1158 if (MI.getOperand(3).getImm() != 0)
1159 return nullptr;
1160
1161 // If we have a zero rotate count, we have:
1162 // M = mask(MB,ME)
1163 // Op0 = (Op1 & ~M) | (Op2 & M)
1164 // Change this to:
1165 // M = mask((ME+1)&31, (MB-1)&31)
1166 // Op0 = (Op2 & ~M) | (Op1 & M)
1167
1168 // Swap op1/op2
1169 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1170 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1171 Register Reg0 = MI.getOperand(0).getReg();
1172 Register Reg1 = MI.getOperand(1).getReg();
1173 Register Reg2 = MI.getOperand(2).getReg();
1174 unsigned SubReg1 = MI.getOperand(1).getSubReg();
1175 unsigned SubReg2 = MI.getOperand(2).getSubReg();
1176 bool Reg1IsKill = MI.getOperand(1).isKill();
1177 bool Reg2IsKill = MI.getOperand(2).isKill();
1178 bool ChangeReg0 = false;
1179 // If machine instrs are no longer in two-address forms, update
1180 // destination register as well.
1181 if (Reg0 == Reg1) {
1182 // Must be two address instruction!
1183 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
1184 "Expecting a two-address instruction!");
1185 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1186 Reg2IsKill = false;
1187 ChangeReg0 = true;
1188 }
1189
1190 // Masks.
1191 unsigned MB = MI.getOperand(4).getImm();
1192 unsigned ME = MI.getOperand(5).getImm();
1193
1194 // We can't commute a trivial mask (there is no way to represent an all-zero
1195 // mask).
1196 if (MB == 0 && ME == 31)
1197 return nullptr;
1198
1199 if (NewMI) {
1200 // Create a new instruction.
1201 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1202 bool Reg0IsDead = MI.getOperand(0).isDead();
1203 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1204 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1205 .addReg(Reg2, getKillRegState(Reg2IsKill))
1206 .addReg(Reg1, getKillRegState(Reg1IsKill))
1207 .addImm((ME + 1) & 31)
1208 .addImm((MB - 1) & 31);
1209 }
1210
1211 if (ChangeReg0) {
1212 MI.getOperand(0).setReg(Reg2);
1213 MI.getOperand(0).setSubReg(SubReg2);
1214 }
1215 MI.getOperand(2).setReg(Reg1);
1216 MI.getOperand(1).setReg(Reg2);
1217 MI.getOperand(2).setSubReg(SubReg1);
1218 MI.getOperand(1).setSubReg(SubReg2);
1219 MI.getOperand(2).setIsKill(Reg1IsKill);
1220 MI.getOperand(1).setIsKill(Reg2IsKill);
1221
1222 // Swap the mask around.
1223 MI.getOperand(4).setImm((ME + 1) & 31);
1224 MI.getOperand(5).setImm((MB - 1) & 31);
1225 return &MI;
1226 }
1227
findCommutedOpIndices(const MachineInstr & MI,unsigned & SrcOpIdx1,unsigned & SrcOpIdx2) const1228 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1229 unsigned &SrcOpIdx1,
1230 unsigned &SrcOpIdx2) const {
1231 // For VSX A-Type FMA instructions, it is the first two operands that can be
1232 // commuted, however, because the non-encoded tied input operand is listed
1233 // first, the operands to swap are actually the second and third.
1234
1235 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1236 if (AltOpc == -1)
1237 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1238
1239 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1240 // and SrcOpIdx2.
1241 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1242 }
1243
insertNoop(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const1244 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
1245 MachineBasicBlock::iterator MI) const {
1246 // This function is used for scheduling, and the nop wanted here is the type
1247 // that terminates dispatch groups on the POWER cores.
1248 unsigned Directive = Subtarget.getCPUDirective();
1249 unsigned Opcode;
1250 switch (Directive) {
1251 default: Opcode = PPC::NOP; break;
1252 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1253 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1254 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1255 // FIXME: Update when POWER9 scheduling model is ready.
1256 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1257 }
1258
1259 DebugLoc DL;
1260 BuildMI(MBB, MI, DL, get(Opcode));
1261 }
1262
1263 /// Return the noop instruction to use for a noop.
getNop() const1264 MCInst PPCInstrInfo::getNop() const {
1265 MCInst Nop;
1266 Nop.setOpcode(PPC::NOP);
1267 return Nop;
1268 }
1269
1270 // Branch analysis.
1271 // Note: If the condition register is set to CTR or CTR8 then this is a
1272 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const1273 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1274 MachineBasicBlock *&TBB,
1275 MachineBasicBlock *&FBB,
1276 SmallVectorImpl<MachineOperand> &Cond,
1277 bool AllowModify) const {
1278 bool isPPC64 = Subtarget.isPPC64();
1279
1280 // If the block has no terminators, it just falls into the block after it.
1281 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1282 if (I == MBB.end())
1283 return false;
1284
1285 if (!isUnpredicatedTerminator(*I))
1286 return false;
1287
1288 if (AllowModify) {
1289 // If the BB ends with an unconditional branch to the fallthrough BB,
1290 // we eliminate the branch instruction.
1291 if (I->getOpcode() == PPC::B &&
1292 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1293 I->eraseFromParent();
1294
1295 // We update iterator after deleting the last branch.
1296 I = MBB.getLastNonDebugInstr();
1297 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1298 return false;
1299 }
1300 }
1301
1302 // Get the last instruction in the block.
1303 MachineInstr &LastInst = *I;
1304
1305 // If there is only one terminator instruction, process it.
1306 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1307 if (LastInst.getOpcode() == PPC::B) {
1308 if (!LastInst.getOperand(0).isMBB())
1309 return true;
1310 TBB = LastInst.getOperand(0).getMBB();
1311 return false;
1312 } else if (LastInst.getOpcode() == PPC::BCC) {
1313 if (!LastInst.getOperand(2).isMBB())
1314 return true;
1315 // Block ends with fall-through condbranch.
1316 TBB = LastInst.getOperand(2).getMBB();
1317 Cond.push_back(LastInst.getOperand(0));
1318 Cond.push_back(LastInst.getOperand(1));
1319 return false;
1320 } else if (LastInst.getOpcode() == PPC::BC) {
1321 if (!LastInst.getOperand(1).isMBB())
1322 return true;
1323 // Block ends with fall-through condbranch.
1324 TBB = LastInst.getOperand(1).getMBB();
1325 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1326 Cond.push_back(LastInst.getOperand(0));
1327 return false;
1328 } else if (LastInst.getOpcode() == PPC::BCn) {
1329 if (!LastInst.getOperand(1).isMBB())
1330 return true;
1331 // Block ends with fall-through condbranch.
1332 TBB = LastInst.getOperand(1).getMBB();
1333 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1334 Cond.push_back(LastInst.getOperand(0));
1335 return false;
1336 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1337 LastInst.getOpcode() == PPC::BDNZ) {
1338 if (!LastInst.getOperand(0).isMBB())
1339 return true;
1340 if (DisableCTRLoopAnal)
1341 return true;
1342 TBB = LastInst.getOperand(0).getMBB();
1343 Cond.push_back(MachineOperand::CreateImm(1));
1344 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1345 true));
1346 return false;
1347 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1348 LastInst.getOpcode() == PPC::BDZ) {
1349 if (!LastInst.getOperand(0).isMBB())
1350 return true;
1351 if (DisableCTRLoopAnal)
1352 return true;
1353 TBB = LastInst.getOperand(0).getMBB();
1354 Cond.push_back(MachineOperand::CreateImm(0));
1355 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1356 true));
1357 return false;
1358 }
1359
1360 // Otherwise, don't know what this is.
1361 return true;
1362 }
1363
1364 // Get the instruction before it if it's a terminator.
1365 MachineInstr &SecondLastInst = *I;
1366
1367 // If there are three terminators, we don't know what sort of block this is.
1368 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1369 return true;
1370
1371 // If the block ends with PPC::B and PPC:BCC, handle it.
1372 if (SecondLastInst.getOpcode() == PPC::BCC &&
1373 LastInst.getOpcode() == PPC::B) {
1374 if (!SecondLastInst.getOperand(2).isMBB() ||
1375 !LastInst.getOperand(0).isMBB())
1376 return true;
1377 TBB = SecondLastInst.getOperand(2).getMBB();
1378 Cond.push_back(SecondLastInst.getOperand(0));
1379 Cond.push_back(SecondLastInst.getOperand(1));
1380 FBB = LastInst.getOperand(0).getMBB();
1381 return false;
1382 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1383 LastInst.getOpcode() == PPC::B) {
1384 if (!SecondLastInst.getOperand(1).isMBB() ||
1385 !LastInst.getOperand(0).isMBB())
1386 return true;
1387 TBB = SecondLastInst.getOperand(1).getMBB();
1388 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1389 Cond.push_back(SecondLastInst.getOperand(0));
1390 FBB = LastInst.getOperand(0).getMBB();
1391 return false;
1392 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1393 LastInst.getOpcode() == PPC::B) {
1394 if (!SecondLastInst.getOperand(1).isMBB() ||
1395 !LastInst.getOperand(0).isMBB())
1396 return true;
1397 TBB = SecondLastInst.getOperand(1).getMBB();
1398 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1399 Cond.push_back(SecondLastInst.getOperand(0));
1400 FBB = LastInst.getOperand(0).getMBB();
1401 return false;
1402 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1403 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1404 LastInst.getOpcode() == PPC::B) {
1405 if (!SecondLastInst.getOperand(0).isMBB() ||
1406 !LastInst.getOperand(0).isMBB())
1407 return true;
1408 if (DisableCTRLoopAnal)
1409 return true;
1410 TBB = SecondLastInst.getOperand(0).getMBB();
1411 Cond.push_back(MachineOperand::CreateImm(1));
1412 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1413 true));
1414 FBB = LastInst.getOperand(0).getMBB();
1415 return false;
1416 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1417 SecondLastInst.getOpcode() == PPC::BDZ) &&
1418 LastInst.getOpcode() == PPC::B) {
1419 if (!SecondLastInst.getOperand(0).isMBB() ||
1420 !LastInst.getOperand(0).isMBB())
1421 return true;
1422 if (DisableCTRLoopAnal)
1423 return true;
1424 TBB = SecondLastInst.getOperand(0).getMBB();
1425 Cond.push_back(MachineOperand::CreateImm(0));
1426 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1427 true));
1428 FBB = LastInst.getOperand(0).getMBB();
1429 return false;
1430 }
1431
1432 // If the block ends with two PPC:Bs, handle it. The second one is not
1433 // executed, so remove it.
1434 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1435 if (!SecondLastInst.getOperand(0).isMBB())
1436 return true;
1437 TBB = SecondLastInst.getOperand(0).getMBB();
1438 I = LastInst;
1439 if (AllowModify)
1440 I->eraseFromParent();
1441 return false;
1442 }
1443
1444 // Otherwise, can't handle this.
1445 return true;
1446 }
1447
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const1448 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1449 int *BytesRemoved) const {
1450 assert(!BytesRemoved && "code size not handled");
1451
1452 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1453 if (I == MBB.end())
1454 return 0;
1455
1456 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1457 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1458 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1459 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1460 return 0;
1461
1462 // Remove the branch.
1463 I->eraseFromParent();
1464
1465 I = MBB.end();
1466
1467 if (I == MBB.begin()) return 1;
1468 --I;
1469 if (I->getOpcode() != PPC::BCC &&
1470 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1471 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1472 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1473 return 1;
1474
1475 // Remove the branch.
1476 I->eraseFromParent();
1477 return 2;
1478 }
1479
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const1480 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1481 MachineBasicBlock *TBB,
1482 MachineBasicBlock *FBB,
1483 ArrayRef<MachineOperand> Cond,
1484 const DebugLoc &DL,
1485 int *BytesAdded) const {
1486 // Shouldn't be a fall through.
1487 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1488 assert((Cond.size() == 2 || Cond.size() == 0) &&
1489 "PPC branch conditions have two components!");
1490 assert(!BytesAdded && "code size not handled");
1491
1492 bool isPPC64 = Subtarget.isPPC64();
1493
1494 // One-way branch.
1495 if (!FBB) {
1496 if (Cond.empty()) // Unconditional branch
1497 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1498 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1499 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1500 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1501 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1502 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1503 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1504 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1505 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1506 else // Conditional branch
1507 BuildMI(&MBB, DL, get(PPC::BCC))
1508 .addImm(Cond[0].getImm())
1509 .add(Cond[1])
1510 .addMBB(TBB);
1511 return 1;
1512 }
1513
1514 // Two-way Conditional Branch.
1515 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1516 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1517 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1518 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1519 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1520 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1521 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1522 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1523 else
1524 BuildMI(&MBB, DL, get(PPC::BCC))
1525 .addImm(Cond[0].getImm())
1526 .add(Cond[1])
1527 .addMBB(TBB);
1528 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1529 return 2;
1530 }
1531
1532 // Select analysis.
canInsertSelect(const MachineBasicBlock & MBB,ArrayRef<MachineOperand> Cond,Register DstReg,Register TrueReg,Register FalseReg,int & CondCycles,int & TrueCycles,int & FalseCycles) const1533 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1534 ArrayRef<MachineOperand> Cond,
1535 Register DstReg, Register TrueReg,
1536 Register FalseReg, int &CondCycles,
1537 int &TrueCycles, int &FalseCycles) const {
1538 if (Cond.size() != 2)
1539 return false;
1540
1541 // If this is really a bdnz-like condition, then it cannot be turned into a
1542 // select.
1543 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1544 return false;
1545
1546 // If the conditional branch uses a physical register, then it cannot be
1547 // turned into a select.
1548 if (Register::isPhysicalRegister(Cond[1].getReg()))
1549 return false;
1550
1551 // Check register classes.
1552 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1553 const TargetRegisterClass *RC =
1554 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1555 if (!RC)
1556 return false;
1557
1558 // isel is for regular integer GPRs only.
1559 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1560 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1561 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1562 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1563 return false;
1564
1565 // FIXME: These numbers are for the A2, how well they work for other cores is
1566 // an open question. On the A2, the isel instruction has a 2-cycle latency
1567 // but single-cycle throughput. These numbers are used in combination with
1568 // the MispredictPenalty setting from the active SchedMachineModel.
1569 CondCycles = 1;
1570 TrueCycles = 1;
1571 FalseCycles = 1;
1572
1573 return true;
1574 }
1575
insertSelect(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & dl,Register DestReg,ArrayRef<MachineOperand> Cond,Register TrueReg,Register FalseReg) const1576 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1577 MachineBasicBlock::iterator MI,
1578 const DebugLoc &dl, Register DestReg,
1579 ArrayRef<MachineOperand> Cond, Register TrueReg,
1580 Register FalseReg) const {
1581 assert(Cond.size() == 2 &&
1582 "PPC branch conditions have two components!");
1583
1584 // Get the register classes.
1585 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1586 const TargetRegisterClass *RC =
1587 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1588 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1589
1590 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1591 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1592 assert((Is64Bit ||
1593 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1594 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1595 "isel is for regular integer GPRs only");
1596
1597 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1598 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1599
1600 unsigned SubIdx = 0;
1601 bool SwapOps = false;
1602 switch (SelectPred) {
1603 case PPC::PRED_EQ:
1604 case PPC::PRED_EQ_MINUS:
1605 case PPC::PRED_EQ_PLUS:
1606 SubIdx = PPC::sub_eq; SwapOps = false; break;
1607 case PPC::PRED_NE:
1608 case PPC::PRED_NE_MINUS:
1609 case PPC::PRED_NE_PLUS:
1610 SubIdx = PPC::sub_eq; SwapOps = true; break;
1611 case PPC::PRED_LT:
1612 case PPC::PRED_LT_MINUS:
1613 case PPC::PRED_LT_PLUS:
1614 SubIdx = PPC::sub_lt; SwapOps = false; break;
1615 case PPC::PRED_GE:
1616 case PPC::PRED_GE_MINUS:
1617 case PPC::PRED_GE_PLUS:
1618 SubIdx = PPC::sub_lt; SwapOps = true; break;
1619 case PPC::PRED_GT:
1620 case PPC::PRED_GT_MINUS:
1621 case PPC::PRED_GT_PLUS:
1622 SubIdx = PPC::sub_gt; SwapOps = false; break;
1623 case PPC::PRED_LE:
1624 case PPC::PRED_LE_MINUS:
1625 case PPC::PRED_LE_PLUS:
1626 SubIdx = PPC::sub_gt; SwapOps = true; break;
1627 case PPC::PRED_UN:
1628 case PPC::PRED_UN_MINUS:
1629 case PPC::PRED_UN_PLUS:
1630 SubIdx = PPC::sub_un; SwapOps = false; break;
1631 case PPC::PRED_NU:
1632 case PPC::PRED_NU_MINUS:
1633 case PPC::PRED_NU_PLUS:
1634 SubIdx = PPC::sub_un; SwapOps = true; break;
1635 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1636 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1637 }
1638
1639 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1640 SecondReg = SwapOps ? TrueReg : FalseReg;
1641
1642 // The first input register of isel cannot be r0. If it is a member
1643 // of a register class that can be r0, then copy it first (the
1644 // register allocator should eliminate the copy).
1645 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1646 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1647 const TargetRegisterClass *FirstRC =
1648 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1649 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1650 Register OldFirstReg = FirstReg;
1651 FirstReg = MRI.createVirtualRegister(FirstRC);
1652 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1653 .addReg(OldFirstReg);
1654 }
1655
1656 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1657 .addReg(FirstReg).addReg(SecondReg)
1658 .addReg(Cond[1].getReg(), 0, SubIdx);
1659 }
1660
getCRBitValue(unsigned CRBit)1661 static unsigned getCRBitValue(unsigned CRBit) {
1662 unsigned Ret = 4;
1663 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1664 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1665 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1666 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1667 Ret = 3;
1668 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1669 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1670 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1671 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1672 Ret = 2;
1673 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1674 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1675 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1676 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1677 Ret = 1;
1678 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1679 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1680 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1681 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1682 Ret = 0;
1683
1684 assert(Ret != 4 && "Invalid CR bit register");
1685 return Ret;
1686 }
1687
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const1688 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1689 MachineBasicBlock::iterator I,
1690 const DebugLoc &DL, MCRegister DestReg,
1691 MCRegister SrcReg, bool KillSrc) const {
1692 // We can end up with self copies and similar things as a result of VSX copy
1693 // legalization. Promote them here.
1694 const TargetRegisterInfo *TRI = &getRegisterInfo();
1695 if (PPC::F8RCRegClass.contains(DestReg) &&
1696 PPC::VSRCRegClass.contains(SrcReg)) {
1697 MCRegister SuperReg =
1698 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1699
1700 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1701 llvm_unreachable("nop VSX copy");
1702
1703 DestReg = SuperReg;
1704 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1705 PPC::VSRCRegClass.contains(DestReg)) {
1706 MCRegister SuperReg =
1707 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1708
1709 if (VSXSelfCopyCrash && DestReg == SuperReg)
1710 llvm_unreachable("nop VSX copy");
1711
1712 SrcReg = SuperReg;
1713 }
1714
1715 // Different class register copy
1716 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1717 PPC::GPRCRegClass.contains(DestReg)) {
1718 MCRegister CRReg = getCRFromCRBit(SrcReg);
1719 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1720 getKillRegState(KillSrc);
1721 // Rotate the CR bit in the CR fields to be the least significant bit and
1722 // then mask with 0x1 (MB = ME = 31).
1723 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1724 .addReg(DestReg, RegState::Kill)
1725 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1726 .addImm(31)
1727 .addImm(31);
1728 return;
1729 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1730 (PPC::G8RCRegClass.contains(DestReg) ||
1731 PPC::GPRCRegClass.contains(DestReg))) {
1732 bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1733 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1734 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1735 unsigned CRNum = TRI->getEncodingValue(SrcReg);
1736 BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1737 getKillRegState(KillSrc);
1738 if (CRNum == 7)
1739 return;
1740 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1741 BuildMI(MBB, I, DL, get(ShCode), DestReg)
1742 .addReg(DestReg, RegState::Kill)
1743 .addImm(CRNum * 4 + 4)
1744 .addImm(28)
1745 .addImm(31);
1746 return;
1747 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1748 PPC::VSFRCRegClass.contains(DestReg)) {
1749 assert(Subtarget.hasDirectMove() &&
1750 "Subtarget doesn't support directmove, don't know how to copy.");
1751 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1752 NumGPRtoVSRSpill++;
1753 getKillRegState(KillSrc);
1754 return;
1755 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1756 PPC::G8RCRegClass.contains(DestReg)) {
1757 assert(Subtarget.hasDirectMove() &&
1758 "Subtarget doesn't support directmove, don't know how to copy.");
1759 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1760 getKillRegState(KillSrc);
1761 return;
1762 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1763 PPC::GPRCRegClass.contains(DestReg)) {
1764 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1765 getKillRegState(KillSrc);
1766 return;
1767 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1768 PPC::SPERCRegClass.contains(DestReg)) {
1769 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1770 getKillRegState(KillSrc);
1771 return;
1772 }
1773
1774 unsigned Opc;
1775 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1776 Opc = PPC::OR;
1777 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1778 Opc = PPC::OR8;
1779 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1780 Opc = PPC::FMR;
1781 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1782 Opc = PPC::MCRF;
1783 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1784 Opc = PPC::VOR;
1785 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1786 // There are two different ways this can be done:
1787 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1788 // issue in VSU pipeline 0.
1789 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1790 // can go to either pipeline.
1791 // We'll always use xxlor here, because in practically all cases where
1792 // copies are generated, they are close enough to some use that the
1793 // lower-latency form is preferable.
1794 Opc = PPC::XXLOR;
1795 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1796 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1797 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1798 else if (Subtarget.pairedVectorMemops() &&
1799 PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1800 if (SrcReg > PPC::VSRp15)
1801 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1802 else
1803 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1804 if (DestReg > PPC::VSRp15)
1805 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1806 else
1807 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1808 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1809 addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1810 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1811 addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1812 return;
1813 }
1814 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1815 Opc = PPC::CROR;
1816 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1817 Opc = PPC::EVOR;
1818 else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1819 PPC::UACCRCRegClass.contains(DestReg)) &&
1820 (PPC::ACCRCRegClass.contains(SrcReg) ||
1821 PPC::UACCRCRegClass.contains(SrcReg))) {
1822 // If primed, de-prime the source register, copy the individual registers
1823 // and prime the destination if needed. The vector subregisters are
1824 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1825 // source is primed, we need to re-prime it after the copy as well.
1826 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1827 bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1828 bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1829 MCRegister VSLSrcReg =
1830 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1831 MCRegister VSLDestReg =
1832 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1833 if (SrcPrimed)
1834 BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1835 for (unsigned Idx = 0; Idx < 4; Idx++)
1836 BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1837 .addReg(VSLSrcReg + Idx)
1838 .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1839 if (DestPrimed)
1840 BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1841 if (SrcPrimed && !KillSrc)
1842 BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1843 return;
1844 } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1845 PPC::G8pRCRegClass.contains(SrcReg)) {
1846 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1847 unsigned DestRegIdx = DestReg - PPC::G8p0;
1848 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1849 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1850 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1851 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1852 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1853 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1854 .addReg(SrcRegSub0)
1855 .addReg(SrcRegSub0, getKillRegState(KillSrc));
1856 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1857 .addReg(SrcRegSub1)
1858 .addReg(SrcRegSub1, getKillRegState(KillSrc));
1859 return;
1860 } else
1861 llvm_unreachable("Impossible reg-to-reg copy");
1862
1863 const MCInstrDesc &MCID = get(Opc);
1864 if (MCID.getNumOperands() == 3)
1865 BuildMI(MBB, I, DL, MCID, DestReg)
1866 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1867 else
1868 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1869 }
1870
getSpillIndex(const TargetRegisterClass * RC) const1871 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1872 int OpcodeIndex = 0;
1873
1874 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1875 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1876 OpcodeIndex = SOK_Int4Spill;
1877 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1878 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1879 OpcodeIndex = SOK_Int8Spill;
1880 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1881 OpcodeIndex = SOK_Float8Spill;
1882 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1883 OpcodeIndex = SOK_Float4Spill;
1884 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1885 OpcodeIndex = SOK_SPESpill;
1886 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1887 OpcodeIndex = SOK_CRSpill;
1888 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1889 OpcodeIndex = SOK_CRBitSpill;
1890 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1891 OpcodeIndex = SOK_VRVectorSpill;
1892 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1893 OpcodeIndex = SOK_VSXVectorSpill;
1894 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1895 OpcodeIndex = SOK_VectorFloat8Spill;
1896 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1897 OpcodeIndex = SOK_VectorFloat4Spill;
1898 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1899 OpcodeIndex = SOK_SpillToVSR;
1900 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1901 assert(Subtarget.pairedVectorMemops() &&
1902 "Register unexpected when paired memops are disabled.");
1903 OpcodeIndex = SOK_AccumulatorSpill;
1904 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1905 assert(Subtarget.pairedVectorMemops() &&
1906 "Register unexpected when paired memops are disabled.");
1907 OpcodeIndex = SOK_UAccumulatorSpill;
1908 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1909 assert(Subtarget.pairedVectorMemops() &&
1910 "Register unexpected when paired memops are disabled.");
1911 OpcodeIndex = SOK_PairedVecSpill;
1912 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1913 OpcodeIndex = SOK_PairedG8Spill;
1914 } else {
1915 llvm_unreachable("Unknown regclass!");
1916 }
1917 return OpcodeIndex;
1918 }
1919
1920 unsigned
getStoreOpcodeForSpill(const TargetRegisterClass * RC) const1921 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1922 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1923 return OpcodesForSpill[getSpillIndex(RC)];
1924 }
1925
1926 unsigned
getLoadOpcodeForSpill(const TargetRegisterClass * RC) const1927 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1928 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1929 return OpcodesForSpill[getSpillIndex(RC)];
1930 }
1931
StoreRegToStackSlot(MachineFunction & MF,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1932 void PPCInstrInfo::StoreRegToStackSlot(
1933 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1934 const TargetRegisterClass *RC,
1935 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1936 unsigned Opcode = getStoreOpcodeForSpill(RC);
1937 DebugLoc DL;
1938
1939 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1940 FuncInfo->setHasSpills();
1941
1942 NewMIs.push_back(addFrameReference(
1943 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1944 FrameIdx));
1945
1946 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1947 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1948 FuncInfo->setSpillsCR();
1949
1950 if (isXFormMemOp(Opcode))
1951 FuncInfo->setHasNonRISpills();
1952 }
1953
storeRegToStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1954 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1955 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1956 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1957 const TargetRegisterInfo *TRI) const {
1958 MachineFunction &MF = *MBB.getParent();
1959 SmallVector<MachineInstr *, 4> NewMIs;
1960
1961 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1962
1963 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1964 MBB.insert(MI, NewMIs[i]);
1965
1966 const MachineFrameInfo &MFI = MF.getFrameInfo();
1967 MachineMemOperand *MMO = MF.getMachineMemOperand(
1968 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1969 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1970 MFI.getObjectAlign(FrameIdx));
1971 NewMIs.back()->addMemOperand(MF, MMO);
1972 }
1973
storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1974 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1975 MachineBasicBlock::iterator MI,
1976 Register SrcReg, bool isKill,
1977 int FrameIdx,
1978 const TargetRegisterClass *RC,
1979 const TargetRegisterInfo *TRI) const {
1980 // We need to avoid a situation in which the value from a VRRC register is
1981 // spilled using an Altivec instruction and reloaded into a VSRC register
1982 // using a VSX instruction. The issue with this is that the VSX
1983 // load/store instructions swap the doublewords in the vector and the Altivec
1984 // ones don't. The register classes on the spill/reload may be different if
1985 // the register is defined using an Altivec instruction and is then used by a
1986 // VSX instruction.
1987 RC = updatedRC(RC);
1988 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1989 }
1990
LoadRegFromStackSlot(MachineFunction & MF,const DebugLoc & DL,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1991 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1992 unsigned DestReg, int FrameIdx,
1993 const TargetRegisterClass *RC,
1994 SmallVectorImpl<MachineInstr *> &NewMIs)
1995 const {
1996 unsigned Opcode = getLoadOpcodeForSpill(RC);
1997 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1998 FrameIdx));
1999 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2000
2001 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
2002 PPC::CRBITRCRegClass.hasSubClassEq(RC))
2003 FuncInfo->setSpillsCR();
2004
2005 if (isXFormMemOp(Opcode))
2006 FuncInfo->setHasNonRISpills();
2007 }
2008
loadRegFromStackSlotNoUpd(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const2009 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
2010 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
2011 int FrameIdx, const TargetRegisterClass *RC,
2012 const TargetRegisterInfo *TRI) const {
2013 MachineFunction &MF = *MBB.getParent();
2014 SmallVector<MachineInstr*, 4> NewMIs;
2015 DebugLoc DL;
2016 if (MI != MBB.end()) DL = MI->getDebugLoc();
2017
2018 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2019 FuncInfo->setHasSpills();
2020
2021 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2022
2023 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
2024 MBB.insert(MI, NewMIs[i]);
2025
2026 const MachineFrameInfo &MFI = MF.getFrameInfo();
2027 MachineMemOperand *MMO = MF.getMachineMemOperand(
2028 MachinePointerInfo::getFixedStack(MF, FrameIdx),
2029 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
2030 MFI.getObjectAlign(FrameIdx));
2031 NewMIs.back()->addMemOperand(MF, MMO);
2032 }
2033
loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,Register DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const2034 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
2035 MachineBasicBlock::iterator MI,
2036 Register DestReg, int FrameIdx,
2037 const TargetRegisterClass *RC,
2038 const TargetRegisterInfo *TRI) const {
2039 // We need to avoid a situation in which the value from a VRRC register is
2040 // spilled using an Altivec instruction and reloaded into a VSRC register
2041 // using a VSX instruction. The issue with this is that the VSX
2042 // load/store instructions swap the doublewords in the vector and the Altivec
2043 // ones don't. The register classes on the spill/reload may be different if
2044 // the register is defined using an Altivec instruction and is then used by a
2045 // VSX instruction.
2046 RC = updatedRC(RC);
2047
2048 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2049 }
2050
2051 bool PPCInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const2052 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
2053 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2054 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2055 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2056 else
2057 // Leave the CR# the same, but invert the condition.
2058 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2059 return false;
2060 }
2061
2062 // For some instructions, it is legal to fold ZERO into the RA register field.
2063 // This function performs that fold by replacing the operand with PPC::ZERO,
2064 // it does not consider whether the load immediate zero is no longer in use.
onlyFoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg) const2065 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2066 Register Reg) const {
2067 // A zero immediate should always be loaded with a single li.
2068 unsigned DefOpc = DefMI.getOpcode();
2069 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2070 return false;
2071 if (!DefMI.getOperand(1).isImm())
2072 return false;
2073 if (DefMI.getOperand(1).getImm() != 0)
2074 return false;
2075
2076 // Note that we cannot here invert the arguments of an isel in order to fold
2077 // a ZERO into what is presented as the second argument. All we have here
2078 // is the condition bit, and that might come from a CR-logical bit operation.
2079
2080 const MCInstrDesc &UseMCID = UseMI.getDesc();
2081
2082 // Only fold into real machine instructions.
2083 if (UseMCID.isPseudo())
2084 return false;
2085
2086 // We need to find which of the User's operands is to be folded, that will be
2087 // the operand that matches the given register ID.
2088 unsigned UseIdx;
2089 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2090 if (UseMI.getOperand(UseIdx).isReg() &&
2091 UseMI.getOperand(UseIdx).getReg() == Reg)
2092 break;
2093
2094 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2095 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2096
2097 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
2098
2099 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2100 // register (which might also be specified as a pointer class kind).
2101 if (UseInfo->isLookupPtrRegClass()) {
2102 if (UseInfo->RegClass /* Kind */ != 1)
2103 return false;
2104 } else {
2105 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2106 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2107 return false;
2108 }
2109
2110 // Make sure this is not tied to an output register (or otherwise
2111 // constrained). This is true for ST?UX registers, for example, which
2112 // are tied to their output registers.
2113 if (UseInfo->Constraints != 0)
2114 return false;
2115
2116 MCRegister ZeroReg;
2117 if (UseInfo->isLookupPtrRegClass()) {
2118 bool isPPC64 = Subtarget.isPPC64();
2119 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2120 } else {
2121 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2122 PPC::ZERO8 : PPC::ZERO;
2123 }
2124
2125 UseMI.getOperand(UseIdx).setReg(ZeroReg);
2126 return true;
2127 }
2128
2129 // Folds zero into instructions which have a load immediate zero as an operand
2130 // but also recognize zero as immediate zero. If the definition of the load
2131 // has no more users it is deleted.
FoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg,MachineRegisterInfo * MRI) const2132 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2133 Register Reg, MachineRegisterInfo *MRI) const {
2134 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2135 if (MRI->use_nodbg_empty(Reg))
2136 DefMI.eraseFromParent();
2137 return Changed;
2138 }
2139
MBBDefinesCTR(MachineBasicBlock & MBB)2140 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
2141 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
2142 I != IE; ++I)
2143 if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8))
2144 return true;
2145 return false;
2146 }
2147
2148 // We should make sure that, if we're going to predicate both sides of a
2149 // condition (a diamond), that both sides don't define the counter register. We
2150 // can predicate counter-decrement-based branches, but while that predicates
2151 // the branching, it does not predicate the counter decrement. If we tried to
2152 // merge the triangle into one predicated block, we'd decrement the counter
2153 // twice.
isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumT,unsigned ExtraT,MachineBasicBlock & FMBB,unsigned NumF,unsigned ExtraF,BranchProbability Probability) const2154 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
2155 unsigned NumT, unsigned ExtraT,
2156 MachineBasicBlock &FMBB,
2157 unsigned NumF, unsigned ExtraF,
2158 BranchProbability Probability) const {
2159 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2160 }
2161
2162
isPredicated(const MachineInstr & MI) const2163 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
2164 // The predicated branches are identified by their type, not really by the
2165 // explicit presence of a predicate. Furthermore, some of them can be
2166 // predicated more than once. Because if conversion won't try to predicate
2167 // any instruction which already claims to be predicated (by returning true
2168 // here), always return false. In doing so, we let isPredicable() be the
2169 // final word on whether not the instruction can be (further) predicated.
2170
2171 return false;
2172 }
2173
isSchedulingBoundary(const MachineInstr & MI,const MachineBasicBlock * MBB,const MachineFunction & MF) const2174 bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2175 const MachineBasicBlock *MBB,
2176 const MachineFunction &MF) const {
2177 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2178 // across them, since some FP operations may change content of FPSCR.
2179 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2180 if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF)
2181 return true;
2182 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
2183 }
2184
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const2185 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
2186 ArrayRef<MachineOperand> Pred) const {
2187 unsigned OpC = MI.getOpcode();
2188 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2189 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2190 bool isPPC64 = Subtarget.isPPC64();
2191 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2192 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2193 // Need add Def and Use for CTR implicit operand.
2194 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2195 .addReg(Pred[1].getReg(), RegState::Implicit)
2196 .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2197 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2198 MI.setDesc(get(PPC::BCLR));
2199 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2200 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2201 MI.setDesc(get(PPC::BCLRn));
2202 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2203 } else {
2204 MI.setDesc(get(PPC::BCCLR));
2205 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2206 .addImm(Pred[0].getImm())
2207 .add(Pred[1]);
2208 }
2209
2210 return true;
2211 } else if (OpC == PPC::B) {
2212 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2213 bool isPPC64 = Subtarget.isPPC64();
2214 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2215 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2216 // Need add Def and Use for CTR implicit operand.
2217 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2218 .addReg(Pred[1].getReg(), RegState::Implicit)
2219 .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2220 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2221 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2222 MI.RemoveOperand(0);
2223
2224 MI.setDesc(get(PPC::BC));
2225 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2226 .add(Pred[1])
2227 .addMBB(MBB);
2228 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2229 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2230 MI.RemoveOperand(0);
2231
2232 MI.setDesc(get(PPC::BCn));
2233 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2234 .add(Pred[1])
2235 .addMBB(MBB);
2236 } else {
2237 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2238 MI.RemoveOperand(0);
2239
2240 MI.setDesc(get(PPC::BCC));
2241 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2242 .addImm(Pred[0].getImm())
2243 .add(Pred[1])
2244 .addMBB(MBB);
2245 }
2246
2247 return true;
2248 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2249 OpC == PPC::BCTRL8) {
2250 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2251 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2252
2253 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8;
2254 bool isPPC64 = Subtarget.isPPC64();
2255
2256 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2257 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2258 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2259 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2260 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2261 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2262 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2263 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2264 } else {
2265 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2266 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2267 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2268 .addImm(Pred[0].getImm())
2269 .add(Pred[1]);
2270 }
2271
2272 // Need add Def and Use for LR implicit operand.
2273 if (setLR)
2274 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2275 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2276 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2277
2278 return true;
2279 }
2280
2281 return false;
2282 }
2283
SubsumesPredicate(ArrayRef<MachineOperand> Pred1,ArrayRef<MachineOperand> Pred2) const2284 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
2285 ArrayRef<MachineOperand> Pred2) const {
2286 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2287 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2288
2289 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2290 return false;
2291 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2292 return false;
2293
2294 // P1 can only subsume P2 if they test the same condition register.
2295 if (Pred1[1].getReg() != Pred2[1].getReg())
2296 return false;
2297
2298 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2299 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2300
2301 if (P1 == P2)
2302 return true;
2303
2304 // Does P1 subsume P2, e.g. GE subsumes GT.
2305 if (P1 == PPC::PRED_LE &&
2306 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2307 return true;
2308 if (P1 == PPC::PRED_GE &&
2309 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2310 return true;
2311
2312 return false;
2313 }
2314
ClobbersPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred,bool SkipDead) const2315 bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
2316 std::vector<MachineOperand> &Pred,
2317 bool SkipDead) const {
2318 // Note: At the present time, the contents of Pred from this function is
2319 // unused by IfConversion. This implementation follows ARM by pushing the
2320 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2321 // predicate, instructions defining CTR or CTR8 are also included as
2322 // predicate-defining instructions.
2323
2324 const TargetRegisterClass *RCs[] =
2325 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2326 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2327
2328 bool Found = false;
2329 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
2330 const MachineOperand &MO = MI.getOperand(i);
2331 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
2332 const TargetRegisterClass *RC = RCs[c];
2333 if (MO.isReg()) {
2334 if (MO.isDef() && RC->contains(MO.getReg())) {
2335 Pred.push_back(MO);
2336 Found = true;
2337 }
2338 } else if (MO.isRegMask()) {
2339 for (TargetRegisterClass::iterator I = RC->begin(),
2340 IE = RC->end(); I != IE; ++I)
2341 if (MO.clobbersPhysReg(*I)) {
2342 Pred.push_back(MO);
2343 Found = true;
2344 }
2345 }
2346 }
2347 }
2348
2349 return Found;
2350 }
2351
analyzeCompare(const MachineInstr & MI,Register & SrcReg,Register & SrcReg2,int64_t & Mask,int64_t & Value) const2352 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2353 Register &SrcReg2, int64_t &Mask,
2354 int64_t &Value) const {
2355 unsigned Opc = MI.getOpcode();
2356
2357 switch (Opc) {
2358 default: return false;
2359 case PPC::CMPWI:
2360 case PPC::CMPLWI:
2361 case PPC::CMPDI:
2362 case PPC::CMPLDI:
2363 SrcReg = MI.getOperand(1).getReg();
2364 SrcReg2 = 0;
2365 Value = MI.getOperand(2).getImm();
2366 Mask = 0xFFFF;
2367 return true;
2368 case PPC::CMPW:
2369 case PPC::CMPLW:
2370 case PPC::CMPD:
2371 case PPC::CMPLD:
2372 case PPC::FCMPUS:
2373 case PPC::FCMPUD:
2374 SrcReg = MI.getOperand(1).getReg();
2375 SrcReg2 = MI.getOperand(2).getReg();
2376 Value = 0;
2377 Mask = 0;
2378 return true;
2379 }
2380 }
2381
optimizeCompareInstr(MachineInstr & CmpInstr,Register SrcReg,Register SrcReg2,int64_t Mask,int64_t Value,const MachineRegisterInfo * MRI) const2382 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
2383 Register SrcReg2, int64_t Mask,
2384 int64_t Value,
2385 const MachineRegisterInfo *MRI) const {
2386 if (DisableCmpOpt)
2387 return false;
2388
2389 int OpC = CmpInstr.getOpcode();
2390 Register CRReg = CmpInstr.getOperand(0).getReg();
2391
2392 // FP record forms set CR1 based on the exception status bits, not a
2393 // comparison with zero.
2394 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2395 return false;
2396
2397 const TargetRegisterInfo *TRI = &getRegisterInfo();
2398 // The record forms set the condition register based on a signed comparison
2399 // with zero (so says the ISA manual). This is not as straightforward as it
2400 // seems, however, because this is always a 64-bit comparison on PPC64, even
2401 // for instructions that are 32-bit in nature (like slw for example).
2402 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2403 // for equality checks (as those don't depend on the sign). On PPC64,
2404 // we are restricted to equality for unsigned 64-bit comparisons and for
2405 // signed 32-bit comparisons the applicability is more restricted.
2406 bool isPPC64 = Subtarget.isPPC64();
2407 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2408 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2409 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2410
2411 // Look through copies unless that gets us to a physical register.
2412 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2413 if (ActualSrc.isVirtual())
2414 SrcReg = ActualSrc;
2415
2416 // Get the unique definition of SrcReg.
2417 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2418 if (!MI) return false;
2419
2420 bool equalityOnly = false;
2421 bool noSub = false;
2422 if (isPPC64) {
2423 if (is32BitSignedCompare) {
2424 // We can perform this optimization only if MI is sign-extending.
2425 if (isSignExtended(*MI))
2426 noSub = true;
2427 else
2428 return false;
2429 } else if (is32BitUnsignedCompare) {
2430 // We can perform this optimization, equality only, if MI is
2431 // zero-extending.
2432 if (isZeroExtended(*MI)) {
2433 noSub = true;
2434 equalityOnly = true;
2435 } else
2436 return false;
2437 } else
2438 equalityOnly = is64BitUnsignedCompare;
2439 } else
2440 equalityOnly = is32BitUnsignedCompare;
2441
2442 if (equalityOnly) {
2443 // We need to check the uses of the condition register in order to reject
2444 // non-equality comparisons.
2445 for (MachineRegisterInfo::use_instr_iterator
2446 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2447 I != IE; ++I) {
2448 MachineInstr *UseMI = &*I;
2449 if (UseMI->getOpcode() == PPC::BCC) {
2450 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2451 unsigned PredCond = PPC::getPredicateCondition(Pred);
2452 // We ignore hint bits when checking for non-equality comparisons.
2453 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2454 return false;
2455 } else if (UseMI->getOpcode() == PPC::ISEL ||
2456 UseMI->getOpcode() == PPC::ISEL8) {
2457 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2458 if (SubIdx != PPC::sub_eq)
2459 return false;
2460 } else
2461 return false;
2462 }
2463 }
2464
2465 MachineBasicBlock::iterator I = CmpInstr;
2466
2467 // Scan forward to find the first use of the compare.
2468 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2469 ++I) {
2470 bool FoundUse = false;
2471 for (MachineRegisterInfo::use_instr_iterator
2472 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2473 J != JE; ++J)
2474 if (&*J == &*I) {
2475 FoundUse = true;
2476 break;
2477 }
2478
2479 if (FoundUse)
2480 break;
2481 }
2482
2483 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
2484 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
2485
2486 // There are two possible candidates which can be changed to set CR[01].
2487 // One is MI, the other is a SUB instruction.
2488 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2489 MachineInstr *Sub = nullptr;
2490 if (SrcReg2 != 0)
2491 // MI is not a candidate for CMPrr.
2492 MI = nullptr;
2493 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2494 // same BB as the comparison. This is to allow the check below to avoid calls
2495 // (and other explicit clobbers); instead we should really check for these
2496 // more explicitly (in at least a few predecessors).
2497 else if (MI->getParent() != CmpInstr.getParent())
2498 return false;
2499 else if (Value != 0) {
2500 // The record-form instructions set CR bit based on signed comparison
2501 // against 0. We try to convert a compare against 1 or -1 into a compare
2502 // against 0 to exploit record-form instructions. For example, we change
2503 // the condition "greater than -1" into "greater than or equal to 0"
2504 // and "less than 1" into "less than or equal to 0".
2505
2506 // Since we optimize comparison based on a specific branch condition,
2507 // we don't optimize if condition code is used by more than once.
2508 if (equalityOnly || !MRI->hasOneUse(CRReg))
2509 return false;
2510
2511 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2512 if (UseMI->getOpcode() != PPC::BCC)
2513 return false;
2514
2515 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2516 unsigned PredCond = PPC::getPredicateCondition(Pred);
2517 unsigned PredHint = PPC::getPredicateHint(Pred);
2518 int16_t Immed = (int16_t)Value;
2519
2520 // When modifying the condition in the predicate, we propagate hint bits
2521 // from the original predicate to the new one.
2522 if (Immed == -1 && PredCond == PPC::PRED_GT)
2523 // We convert "greater than -1" into "greater than or equal to 0",
2524 // since we are assuming signed comparison by !equalityOnly
2525 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2526 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2527 // We convert "less than or equal to -1" into "less than 0".
2528 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2529 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2530 // We convert "less than 1" into "less than or equal to 0".
2531 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2532 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2533 // We convert "greater than or equal to 1" into "greater than 0".
2534 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2535 else
2536 return false;
2537
2538 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
2539 }
2540
2541 // Search for Sub.
2542 --I;
2543
2544 // Get ready to iterate backward from CmpInstr.
2545 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2546
2547 for (; I != E && !noSub; --I) {
2548 const MachineInstr &Instr = *I;
2549 unsigned IOpC = Instr.getOpcode();
2550
2551 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2552 Instr.readsRegister(PPC::CR0, TRI)))
2553 // This instruction modifies or uses the record condition register after
2554 // the one we want to change. While we could do this transformation, it
2555 // would likely not be profitable. This transformation removes one
2556 // instruction, and so even forcing RA to generate one move probably
2557 // makes it unprofitable.
2558 return false;
2559
2560 // Check whether CmpInstr can be made redundant by the current instruction.
2561 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2562 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2563 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2564 ((Instr.getOperand(1).getReg() == SrcReg &&
2565 Instr.getOperand(2).getReg() == SrcReg2) ||
2566 (Instr.getOperand(1).getReg() == SrcReg2 &&
2567 Instr.getOperand(2).getReg() == SrcReg))) {
2568 Sub = &*I;
2569 break;
2570 }
2571
2572 if (I == B)
2573 // The 'and' is below the comparison instruction.
2574 return false;
2575 }
2576
2577 // Return false if no candidates exist.
2578 if (!MI && !Sub)
2579 return false;
2580
2581 // The single candidate is called MI.
2582 if (!MI) MI = Sub;
2583
2584 int NewOpC = -1;
2585 int MIOpC = MI->getOpcode();
2586 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2587 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2588 NewOpC = MIOpC;
2589 else {
2590 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2591 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2592 NewOpC = MIOpC;
2593 }
2594
2595 // FIXME: On the non-embedded POWER architectures, only some of the record
2596 // forms are fast, and we should use only the fast ones.
2597
2598 // The defining instruction has a record form (or is already a record
2599 // form). It is possible, however, that we'll need to reverse the condition
2600 // code of the users.
2601 if (NewOpC == -1)
2602 return false;
2603
2604 // This transformation should not be performed if `nsw` is missing and is not
2605 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2606 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2607 // CRReg can reflect if compared values are equal, this optz is still valid.
2608 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2609 Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2610 return false;
2611
2612 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2613 // needs to be updated to be based on SUB. Push the condition code
2614 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2615 // condition code of these operands will be modified.
2616 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2617 // comparison against 0, which may modify predicate.
2618 bool ShouldSwap = false;
2619 if (Sub && Value == 0) {
2620 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2621 Sub->getOperand(2).getReg() == SrcReg;
2622
2623 // The operands to subf are the opposite of sub, so only in the fixed-point
2624 // case, invert the order.
2625 ShouldSwap = !ShouldSwap;
2626 }
2627
2628 if (ShouldSwap)
2629 for (MachineRegisterInfo::use_instr_iterator
2630 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2631 I != IE; ++I) {
2632 MachineInstr *UseMI = &*I;
2633 if (UseMI->getOpcode() == PPC::BCC) {
2634 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2635 unsigned PredCond = PPC::getPredicateCondition(Pred);
2636 assert((!equalityOnly ||
2637 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2638 "Invalid predicate for equality-only optimization");
2639 (void)PredCond; // To suppress warning in release build.
2640 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2641 PPC::getSwappedPredicate(Pred)));
2642 } else if (UseMI->getOpcode() == PPC::ISEL ||
2643 UseMI->getOpcode() == PPC::ISEL8) {
2644 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2645 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2646 "Invalid CR bit for equality-only optimization");
2647
2648 if (NewSubReg == PPC::sub_lt)
2649 NewSubReg = PPC::sub_gt;
2650 else if (NewSubReg == PPC::sub_gt)
2651 NewSubReg = PPC::sub_lt;
2652
2653 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2654 NewSubReg));
2655 } else // We need to abort on a user we don't understand.
2656 return false;
2657 }
2658 assert(!(Value != 0 && ShouldSwap) &&
2659 "Non-zero immediate support and ShouldSwap"
2660 "may conflict in updating predicate");
2661
2662 // Create a new virtual register to hold the value of the CR set by the
2663 // record-form instruction. If the instruction was not previously in
2664 // record form, then set the kill flag on the CR.
2665 CmpInstr.eraseFromParent();
2666
2667 MachineBasicBlock::iterator MII = MI;
2668 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2669 get(TargetOpcode::COPY), CRReg)
2670 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2671
2672 // Even if CR0 register were dead before, it is alive now since the
2673 // instruction we just built uses it.
2674 MI->clearRegisterDeads(PPC::CR0);
2675
2676 if (MIOpC != NewOpC) {
2677 // We need to be careful here: we're replacing one instruction with
2678 // another, and we need to make sure that we get all of the right
2679 // implicit uses and defs. On the other hand, the caller may be holding
2680 // an iterator to this instruction, and so we can't delete it (this is
2681 // specifically the case if this is the instruction directly after the
2682 // compare).
2683
2684 // Rotates are expensive instructions. If we're emitting a record-form
2685 // rotate that can just be an andi/andis, we should just emit that.
2686 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2687 Register GPRRes = MI->getOperand(0).getReg();
2688 int64_t SH = MI->getOperand(2).getImm();
2689 int64_t MB = MI->getOperand(3).getImm();
2690 int64_t ME = MI->getOperand(4).getImm();
2691 // We can only do this if both the start and end of the mask are in the
2692 // same halfword.
2693 bool MBInLoHWord = MB >= 16;
2694 bool MEInLoHWord = ME >= 16;
2695 uint64_t Mask = ~0LLU;
2696
2697 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2698 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2699 // The mask value needs to shift right 16 if we're emitting andis.
2700 Mask >>= MBInLoHWord ? 0 : 16;
2701 NewOpC = MIOpC == PPC::RLWINM
2702 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2703 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2704 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2705 (ME - MB + 1 == SH) && (MB >= 16)) {
2706 // If we are rotating by the exact number of bits as are in the mask
2707 // and the mask is in the least significant bits of the register,
2708 // that's just an andis. (as long as the GPR result has no uses).
2709 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2710 Mask >>= 16;
2711 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2712 }
2713 // If we've set the mask, we can transform.
2714 if (Mask != ~0LLU) {
2715 MI->RemoveOperand(4);
2716 MI->RemoveOperand(3);
2717 MI->getOperand(2).setImm(Mask);
2718 NumRcRotatesConvertedToRcAnd++;
2719 }
2720 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2721 int64_t MB = MI->getOperand(3).getImm();
2722 if (MB >= 48) {
2723 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2724 NewOpC = PPC::ANDI8_rec;
2725 MI->RemoveOperand(3);
2726 MI->getOperand(2).setImm(Mask);
2727 NumRcRotatesConvertedToRcAnd++;
2728 }
2729 }
2730
2731 const MCInstrDesc &NewDesc = get(NewOpC);
2732 MI->setDesc(NewDesc);
2733
2734 if (NewDesc.ImplicitDefs)
2735 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2736 *ImpDefs; ++ImpDefs)
2737 if (!MI->definesRegister(*ImpDefs))
2738 MI->addOperand(*MI->getParent()->getParent(),
2739 MachineOperand::CreateReg(*ImpDefs, true, true));
2740 if (NewDesc.ImplicitUses)
2741 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2742 *ImpUses; ++ImpUses)
2743 if (!MI->readsRegister(*ImpUses))
2744 MI->addOperand(*MI->getParent()->getParent(),
2745 MachineOperand::CreateReg(*ImpUses, false, true));
2746 }
2747 assert(MI->definesRegister(PPC::CR0) &&
2748 "Record-form instruction does not define cr0?");
2749
2750 // Modify the condition code of operands in OperandsToUpdate.
2751 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2752 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2753 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2754 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2755
2756 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2757 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2758
2759 return true;
2760 }
2761
getMemOperandsWithOffsetWidth(const MachineInstr & LdSt,SmallVectorImpl<const MachineOperand * > & BaseOps,int64_t & Offset,bool & OffsetIsScalable,unsigned & Width,const TargetRegisterInfo * TRI) const2762 bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
2763 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2764 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2765 const TargetRegisterInfo *TRI) const {
2766 const MachineOperand *BaseOp;
2767 OffsetIsScalable = false;
2768 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2769 return false;
2770 BaseOps.push_back(BaseOp);
2771 return true;
2772 }
2773
isLdStSafeToCluster(const MachineInstr & LdSt,const TargetRegisterInfo * TRI)2774 static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2775 const TargetRegisterInfo *TRI) {
2776 // If this is a volatile load/store, don't mess with it.
2777 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2778 return false;
2779
2780 if (LdSt.getOperand(2).isFI())
2781 return true;
2782
2783 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2784 // Can't cluster if the instruction modifies the base register
2785 // or it is update form. e.g. ld r2,3(r2)
2786 if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2787 return false;
2788
2789 return true;
2790 }
2791
2792 // Only cluster instruction pair that have the same opcode, and they are
2793 // clusterable according to PowerPC specification.
isClusterableLdStOpcPair(unsigned FirstOpc,unsigned SecondOpc,const PPCSubtarget & Subtarget)2794 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2795 const PPCSubtarget &Subtarget) {
2796 switch (FirstOpc) {
2797 default:
2798 return false;
2799 case PPC::STD:
2800 case PPC::STFD:
2801 case PPC::STXSD:
2802 case PPC::DFSTOREf64:
2803 return FirstOpc == SecondOpc;
2804 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2805 // 32bit and 64bit instruction selection. They are clusterable pair though
2806 // they are different opcode.
2807 case PPC::STW:
2808 case PPC::STW8:
2809 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2810 }
2811 }
2812
shouldClusterMemOps(ArrayRef<const MachineOperand * > BaseOps1,ArrayRef<const MachineOperand * > BaseOps2,unsigned NumLoads,unsigned NumBytes) const2813 bool PPCInstrInfo::shouldClusterMemOps(
2814 ArrayRef<const MachineOperand *> BaseOps1,
2815 ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2816 unsigned NumBytes) const {
2817
2818 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2819 const MachineOperand &BaseOp1 = *BaseOps1.front();
2820 const MachineOperand &BaseOp2 = *BaseOps2.front();
2821 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2822 "Only base registers and frame indices are supported.");
2823
2824 // The NumLoads means the number of loads that has been clustered.
2825 // Don't cluster memory op if there are already two ops clustered at least.
2826 if (NumLoads > 2)
2827 return false;
2828
2829 // Cluster the load/store only when they have the same base
2830 // register or FI.
2831 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2832 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2833 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2834 return false;
2835
2836 // Check if the load/store are clusterable according to the PowerPC
2837 // specification.
2838 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2839 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2840 unsigned FirstOpc = FirstLdSt.getOpcode();
2841 unsigned SecondOpc = SecondLdSt.getOpcode();
2842 const TargetRegisterInfo *TRI = &getRegisterInfo();
2843 // Cluster the load/store only when they have the same opcode, and they are
2844 // clusterable opcode according to PowerPC specification.
2845 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2846 return false;
2847
2848 // Can't cluster load/store that have ordered or volatile memory reference.
2849 if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2850 !isLdStSafeToCluster(SecondLdSt, TRI))
2851 return false;
2852
2853 int64_t Offset1 = 0, Offset2 = 0;
2854 unsigned Width1 = 0, Width2 = 0;
2855 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2856 if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2857 !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2858 Width1 != Width2)
2859 return false;
2860
2861 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2862 "getMemOperandWithOffsetWidth return incorrect base op");
2863 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2864 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2865 return Offset1 + Width1 == Offset2;
2866 }
2867
2868 /// GetInstSize - Return the number of bytes of code the specified
2869 /// instruction may be. This returns the maximum number of bytes.
2870 ///
getInstSizeInBytes(const MachineInstr & MI) const2871 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2872 unsigned Opcode = MI.getOpcode();
2873
2874 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2875 const MachineFunction *MF = MI.getParent()->getParent();
2876 const char *AsmStr = MI.getOperand(0).getSymbolName();
2877 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2878 } else if (Opcode == TargetOpcode::STACKMAP) {
2879 StackMapOpers Opers(&MI);
2880 return Opers.getNumPatchBytes();
2881 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2882 PatchPointOpers Opers(&MI);
2883 return Opers.getNumPatchBytes();
2884 } else {
2885 return get(Opcode).getSize();
2886 }
2887 }
2888
2889 std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const2890 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2891 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2892 return std::make_pair(TF & Mask, TF & ~Mask);
2893 }
2894
2895 ArrayRef<std::pair<unsigned, const char *>>
getSerializableDirectMachineOperandTargetFlags() const2896 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2897 using namespace PPCII;
2898 static const std::pair<unsigned, const char *> TargetFlags[] = {
2899 {MO_LO, "ppc-lo"},
2900 {MO_HA, "ppc-ha"},
2901 {MO_TPREL_LO, "ppc-tprel-lo"},
2902 {MO_TPREL_HA, "ppc-tprel-ha"},
2903 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2904 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2905 {MO_TOC_LO, "ppc-toc-lo"},
2906 {MO_TLS, "ppc-tls"}};
2907 return makeArrayRef(TargetFlags);
2908 }
2909
2910 ArrayRef<std::pair<unsigned, const char *>>
getSerializableBitmaskMachineOperandTargetFlags() const2911 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
2912 using namespace PPCII;
2913 static const std::pair<unsigned, const char *> TargetFlags[] = {
2914 {MO_PLT, "ppc-plt"},
2915 {MO_PIC_FLAG, "ppc-pic"},
2916 {MO_PCREL_FLAG, "ppc-pcrel"},
2917 {MO_GOT_FLAG, "ppc-got"},
2918 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
2919 {MO_TLSGD_FLAG, "ppc-tlsgd"},
2920 {MO_TLSLD_FLAG, "ppc-tlsld"},
2921 {MO_TPREL_FLAG, "ppc-tprel"},
2922 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
2923 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
2924 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
2925 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}};
2926 return makeArrayRef(TargetFlags);
2927 }
2928
2929 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2930 // The VSX versions have the advantage of a full 64-register target whereas
2931 // the FP ones have the advantage of lower latency and higher throughput. So
2932 // what we are after is using the faster instructions in low register pressure
2933 // situations and using the larger register file in high register pressure
2934 // situations.
expandVSXMemPseudo(MachineInstr & MI) const2935 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2936 unsigned UpperOpcode, LowerOpcode;
2937 switch (MI.getOpcode()) {
2938 case PPC::DFLOADf32:
2939 UpperOpcode = PPC::LXSSP;
2940 LowerOpcode = PPC::LFS;
2941 break;
2942 case PPC::DFLOADf64:
2943 UpperOpcode = PPC::LXSD;
2944 LowerOpcode = PPC::LFD;
2945 break;
2946 case PPC::DFSTOREf32:
2947 UpperOpcode = PPC::STXSSP;
2948 LowerOpcode = PPC::STFS;
2949 break;
2950 case PPC::DFSTOREf64:
2951 UpperOpcode = PPC::STXSD;
2952 LowerOpcode = PPC::STFD;
2953 break;
2954 case PPC::XFLOADf32:
2955 UpperOpcode = PPC::LXSSPX;
2956 LowerOpcode = PPC::LFSX;
2957 break;
2958 case PPC::XFLOADf64:
2959 UpperOpcode = PPC::LXSDX;
2960 LowerOpcode = PPC::LFDX;
2961 break;
2962 case PPC::XFSTOREf32:
2963 UpperOpcode = PPC::STXSSPX;
2964 LowerOpcode = PPC::STFSX;
2965 break;
2966 case PPC::XFSTOREf64:
2967 UpperOpcode = PPC::STXSDX;
2968 LowerOpcode = PPC::STFDX;
2969 break;
2970 case PPC::LIWAX:
2971 UpperOpcode = PPC::LXSIWAX;
2972 LowerOpcode = PPC::LFIWAX;
2973 break;
2974 case PPC::LIWZX:
2975 UpperOpcode = PPC::LXSIWZX;
2976 LowerOpcode = PPC::LFIWZX;
2977 break;
2978 case PPC::STIWX:
2979 UpperOpcode = PPC::STXSIWX;
2980 LowerOpcode = PPC::STFIWX;
2981 break;
2982 default:
2983 llvm_unreachable("Unknown Operation!");
2984 }
2985
2986 Register TargetReg = MI.getOperand(0).getReg();
2987 unsigned Opcode;
2988 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2989 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2990 Opcode = LowerOpcode;
2991 else
2992 Opcode = UpperOpcode;
2993 MI.setDesc(get(Opcode));
2994 return true;
2995 }
2996
isAnImmediateOperand(const MachineOperand & MO)2997 static bool isAnImmediateOperand(const MachineOperand &MO) {
2998 return MO.isCPI() || MO.isGlobal() || MO.isImm();
2999 }
3000
expandPostRAPseudo(MachineInstr & MI) const3001 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
3002 auto &MBB = *MI.getParent();
3003 auto DL = MI.getDebugLoc();
3004
3005 switch (MI.getOpcode()) {
3006 case PPC::BUILD_UACC: {
3007 MCRegister ACC = MI.getOperand(0).getReg();
3008 MCRegister UACC = MI.getOperand(1).getReg();
3009 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3010 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3011 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3012 // FIXME: This can easily be improved to look up to the top of the MBB
3013 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3014 // we can just re-target any such XXLOR's to DstVSR + offset.
3015 for (int VecNo = 0; VecNo < 4; VecNo++)
3016 BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3017 .addReg(SrcVSR + VecNo)
3018 .addReg(SrcVSR + VecNo);
3019 }
3020 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3021 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3022 // with a NOP.
3023 LLVM_FALLTHROUGH;
3024 }
3025 case PPC::KILL_PAIR: {
3026 MI.setDesc(get(PPC::UNENCODED_NOP));
3027 MI.RemoveOperand(1);
3028 MI.RemoveOperand(0);
3029 return true;
3030 }
3031 case TargetOpcode::LOAD_STACK_GUARD: {
3032 assert(Subtarget.isTargetLinux() &&
3033 "Only Linux target is expected to contain LOAD_STACK_GUARD");
3034 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3035 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3036 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3037 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3038 .addImm(Offset)
3039 .addReg(Reg);
3040 return true;
3041 }
3042 case PPC::DFLOADf32:
3043 case PPC::DFLOADf64:
3044 case PPC::DFSTOREf32:
3045 case PPC::DFSTOREf64: {
3046 assert(Subtarget.hasP9Vector() &&
3047 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3048 assert(MI.getOperand(2).isReg() &&
3049 isAnImmediateOperand(MI.getOperand(1)) &&
3050 "D-form op must have register and immediate operands");
3051 return expandVSXMemPseudo(MI);
3052 }
3053 case PPC::XFLOADf32:
3054 case PPC::XFSTOREf32:
3055 case PPC::LIWAX:
3056 case PPC::LIWZX:
3057 case PPC::STIWX: {
3058 assert(Subtarget.hasP8Vector() &&
3059 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3060 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3061 "X-form op must have register and register operands");
3062 return expandVSXMemPseudo(MI);
3063 }
3064 case PPC::XFLOADf64:
3065 case PPC::XFSTOREf64: {
3066 assert(Subtarget.hasVSX() &&
3067 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3068 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3069 "X-form op must have register and register operands");
3070 return expandVSXMemPseudo(MI);
3071 }
3072 case PPC::SPILLTOVSR_LD: {
3073 Register TargetReg = MI.getOperand(0).getReg();
3074 if (PPC::VSFRCRegClass.contains(TargetReg)) {
3075 MI.setDesc(get(PPC::DFLOADf64));
3076 return expandPostRAPseudo(MI);
3077 }
3078 else
3079 MI.setDesc(get(PPC::LD));
3080 return true;
3081 }
3082 case PPC::SPILLTOVSR_ST: {
3083 Register SrcReg = MI.getOperand(0).getReg();
3084 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3085 NumStoreSPILLVSRRCAsVec++;
3086 MI.setDesc(get(PPC::DFSTOREf64));
3087 return expandPostRAPseudo(MI);
3088 } else {
3089 NumStoreSPILLVSRRCAsGpr++;
3090 MI.setDesc(get(PPC::STD));
3091 }
3092 return true;
3093 }
3094 case PPC::SPILLTOVSR_LDX: {
3095 Register TargetReg = MI.getOperand(0).getReg();
3096 if (PPC::VSFRCRegClass.contains(TargetReg))
3097 MI.setDesc(get(PPC::LXSDX));
3098 else
3099 MI.setDesc(get(PPC::LDX));
3100 return true;
3101 }
3102 case PPC::SPILLTOVSR_STX: {
3103 Register SrcReg = MI.getOperand(0).getReg();
3104 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3105 NumStoreSPILLVSRRCAsVec++;
3106 MI.setDesc(get(PPC::STXSDX));
3107 } else {
3108 NumStoreSPILLVSRRCAsGpr++;
3109 MI.setDesc(get(PPC::STDX));
3110 }
3111 return true;
3112 }
3113
3114 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3115 case PPC::CFENCE8: {
3116 auto Val = MI.getOperand(0).getReg();
3117 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
3118 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3119 .addImm(PPC::PRED_NE_MINUS)
3120 .addReg(PPC::CR7)
3121 .addImm(1);
3122 MI.setDesc(get(PPC::ISYNC));
3123 MI.RemoveOperand(0);
3124 return true;
3125 }
3126 }
3127 return false;
3128 }
3129
3130 // Essentially a compile-time implementation of a compare->isel sequence.
3131 // It takes two constants to compare, along with the true/false registers
3132 // and the comparison type (as a subreg to a CR field) and returns one
3133 // of the true/false registers, depending on the comparison results.
selectReg(int64_t Imm1,int64_t Imm2,unsigned CompareOpc,unsigned TrueReg,unsigned FalseReg,unsigned CRSubReg)3134 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3135 unsigned TrueReg, unsigned FalseReg,
3136 unsigned CRSubReg) {
3137 // Signed comparisons. The immediates are assumed to be sign-extended.
3138 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3139 switch (CRSubReg) {
3140 default: llvm_unreachable("Unknown integer comparison type.");
3141 case PPC::sub_lt:
3142 return Imm1 < Imm2 ? TrueReg : FalseReg;
3143 case PPC::sub_gt:
3144 return Imm1 > Imm2 ? TrueReg : FalseReg;
3145 case PPC::sub_eq:
3146 return Imm1 == Imm2 ? TrueReg : FalseReg;
3147 }
3148 }
3149 // Unsigned comparisons.
3150 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3151 switch (CRSubReg) {
3152 default: llvm_unreachable("Unknown integer comparison type.");
3153 case PPC::sub_lt:
3154 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3155 case PPC::sub_gt:
3156 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3157 case PPC::sub_eq:
3158 return Imm1 == Imm2 ? TrueReg : FalseReg;
3159 }
3160 }
3161 return PPC::NoRegister;
3162 }
3163
replaceInstrOperandWithImm(MachineInstr & MI,unsigned OpNo,int64_t Imm) const3164 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
3165 unsigned OpNo,
3166 int64_t Imm) const {
3167 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3168 // Replace the REG with the Immediate.
3169 Register InUseReg = MI.getOperand(OpNo).getReg();
3170 MI.getOperand(OpNo).ChangeToImmediate(Imm);
3171
3172 // We need to make sure that the MI didn't have any implicit use
3173 // of this REG any more. We don't call MI.implicit_operands().empty() to
3174 // return early, since MI's MCID might be changed in calling context, as a
3175 // result its number of explicit operands may be changed, thus the begin of
3176 // implicit operand is changed.
3177 const TargetRegisterInfo *TRI = &getRegisterInfo();
3178 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3179 if (UseOpIdx >= 0) {
3180 MachineOperand &MO = MI.getOperand(UseOpIdx);
3181 if (MO.isImplicit())
3182 // The operands must always be in the following order:
3183 // - explicit reg defs,
3184 // - other explicit operands (reg uses, immediates, etc.),
3185 // - implicit reg defs
3186 // - implicit reg uses
3187 // Therefore, removing the implicit operand won't change the explicit
3188 // operands layout.
3189 MI.RemoveOperand(UseOpIdx);
3190 }
3191 }
3192
3193 // Replace an instruction with one that materializes a constant (and sets
3194 // CR0 if the original instruction was a record-form instruction).
replaceInstrWithLI(MachineInstr & MI,const LoadImmediateInfo & LII) const3195 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
3196 const LoadImmediateInfo &LII) const {
3197 // Remove existing operands.
3198 int OperandToKeep = LII.SetCR ? 1 : 0;
3199 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3200 MI.RemoveOperand(i);
3201
3202 // Replace the instruction.
3203 if (LII.SetCR) {
3204 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3205 // Set the immediate.
3206 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3207 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3208 return;
3209 }
3210 else
3211 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3212
3213 // Set the immediate.
3214 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3215 .addImm(LII.Imm);
3216 }
3217
getDefMIPostRA(unsigned Reg,MachineInstr & MI,bool & SeenIntermediateUse) const3218 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
3219 bool &SeenIntermediateUse) const {
3220 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3221 "Should be called after register allocation.");
3222 const TargetRegisterInfo *TRI = &getRegisterInfo();
3223 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3224 It++;
3225 SeenIntermediateUse = false;
3226 for (; It != E; ++It) {
3227 if (It->modifiesRegister(Reg, TRI))
3228 return &*It;
3229 if (It->readsRegister(Reg, TRI))
3230 SeenIntermediateUse = true;
3231 }
3232 return nullptr;
3233 }
3234
getForwardingDefMI(MachineInstr & MI,unsigned & OpNoForForwarding,bool & SeenIntermediateUse) const3235 MachineInstr *PPCInstrInfo::getForwardingDefMI(
3236 MachineInstr &MI,
3237 unsigned &OpNoForForwarding,
3238 bool &SeenIntermediateUse) const {
3239 OpNoForForwarding = ~0U;
3240 MachineInstr *DefMI = nullptr;
3241 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3242 const TargetRegisterInfo *TRI = &getRegisterInfo();
3243 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3244 // within the basic block to see if the register is defined using an
3245 // LI/LI8/ADDI/ADDI8.
3246 if (MRI->isSSA()) {
3247 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3248 if (!MI.getOperand(i).isReg())
3249 continue;
3250 Register Reg = MI.getOperand(i).getReg();
3251 if (!Register::isVirtualRegister(Reg))
3252 continue;
3253 unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3254 if (Register::isVirtualRegister(TrueReg)) {
3255 DefMI = MRI->getVRegDef(TrueReg);
3256 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
3257 DefMI->getOpcode() == PPC::ADDI ||
3258 DefMI->getOpcode() == PPC::ADDI8) {
3259 OpNoForForwarding = i;
3260 // The ADDI and LI operand maybe exist in one instruction at same
3261 // time. we prefer to fold LI operand as LI only has one Imm operand
3262 // and is more possible to be converted. So if current DefMI is
3263 // ADDI/ADDI8, we continue to find possible LI/LI8.
3264 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3265 break;
3266 }
3267 }
3268 }
3269 } else {
3270 // Looking back through the definition for each operand could be expensive,
3271 // so exit early if this isn't an instruction that either has an immediate
3272 // form or is already an immediate form that we can handle.
3273 ImmInstrInfo III;
3274 unsigned Opc = MI.getOpcode();
3275 bool ConvertibleImmForm =
3276 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3277 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3278 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3279 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3280 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3281 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3282 Opc == PPC::RLWINM8_rec;
3283 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3284 ? isVFRegister(MI.getOperand(0).getReg())
3285 : false;
3286 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3287 return nullptr;
3288
3289 // Don't convert or %X, %Y, %Y since that's just a register move.
3290 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3291 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3292 return nullptr;
3293 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3294 MachineOperand &MO = MI.getOperand(i);
3295 SeenIntermediateUse = false;
3296 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3297 Register Reg = MI.getOperand(i).getReg();
3298 // If we see another use of this reg between the def and the MI,
3299 // we want to flat it so the def isn't deleted.
3300 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3301 if (DefMI) {
3302 // Is this register defined by some form of add-immediate (including
3303 // load-immediate) within this basic block?
3304 switch (DefMI->getOpcode()) {
3305 default:
3306 break;
3307 case PPC::LI:
3308 case PPC::LI8:
3309 case PPC::ADDItocL:
3310 case PPC::ADDI:
3311 case PPC::ADDI8:
3312 OpNoForForwarding = i;
3313 return DefMI;
3314 }
3315 }
3316 }
3317 }
3318 }
3319 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3320 }
3321
getSpillTarget() const3322 unsigned PPCInstrInfo::getSpillTarget() const {
3323 // With P10, we may need to spill paired vector registers or accumulator
3324 // registers. MMA implies paired vectors, so we can just check that.
3325 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3326 return IsP10Variant ? 2 : Subtarget.hasP9Vector() ? 1 : 0;
3327 }
3328
getStoreOpcodesForSpillArray() const3329 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3330 return StoreSpillOpcodesArray[getSpillTarget()];
3331 }
3332
getLoadOpcodesForSpillArray() const3333 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3334 return LoadSpillOpcodesArray[getSpillTarget()];
3335 }
3336
fixupIsDeadOrKill(MachineInstr * StartMI,MachineInstr * EndMI,unsigned RegNo) const3337 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI,
3338 unsigned RegNo) const {
3339 // Conservatively clear kill flag for the register if the instructions are in
3340 // different basic blocks and in SSA form, because the kill flag may no longer
3341 // be right. There is no need to bother with dead flags since defs with no
3342 // uses will be handled by DCE.
3343 MachineRegisterInfo &MRI = StartMI->getParent()->getParent()->getRegInfo();
3344 if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
3345 MRI.clearKillFlags(RegNo);
3346 return;
3347 }
3348
3349 // Instructions between [StartMI, EndMI] should be in same basic block.
3350 assert((StartMI->getParent() == EndMI->getParent()) &&
3351 "Instructions are not in same basic block");
3352
3353 // If before RA, StartMI may be def through COPY, we need to adjust it to the
3354 // real def. See function getForwardingDefMI.
3355 if (MRI.isSSA()) {
3356 bool Reads, Writes;
3357 std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo);
3358 if (!Reads && !Writes) {
3359 assert(Register::isVirtualRegister(RegNo) &&
3360 "Must be a virtual register");
3361 // Get real def and ignore copies.
3362 StartMI = MRI.getVRegDef(RegNo);
3363 }
3364 }
3365
3366 bool IsKillSet = false;
3367
3368 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
3369 MachineOperand &MO = MI.getOperand(Index);
3370 if (MO.isReg() && MO.isUse() && MO.isKill() &&
3371 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
3372 MO.setIsKill(false);
3373 };
3374
3375 // Set killed flag for EndMI.
3376 // No need to do anything if EndMI defines RegNo.
3377 int UseIndex =
3378 EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
3379 if (UseIndex != -1) {
3380 EndMI->getOperand(UseIndex).setIsKill(true);
3381 IsKillSet = true;
3382 // Clear killed flag for other EndMI operands related to RegNo. In some
3383 // upexpected cases, killed may be set multiple times for same register
3384 // operand in same MI.
3385 for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
3386 if (i != UseIndex)
3387 clearOperandKillInfo(*EndMI, i);
3388 }
3389
3390 // Walking the inst in reverse order (EndMI -> StartMI].
3391 MachineBasicBlock::reverse_iterator It = *EndMI;
3392 MachineBasicBlock::reverse_iterator E = EndMI->getParent()->rend();
3393 // EndMI has been handled above, skip it here.
3394 It++;
3395 MachineOperand *MO = nullptr;
3396 for (; It != E; ++It) {
3397 // Skip insturctions which could not be a def/use of RegNo.
3398 if (It->isDebugInstr() || It->isPosition())
3399 continue;
3400
3401 // Clear killed flag for all It operands related to RegNo. In some
3402 // upexpected cases, killed may be set multiple times for same register
3403 // operand in same MI.
3404 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
3405 clearOperandKillInfo(*It, i);
3406
3407 // If killed is not set, set killed for its last use or set dead for its def
3408 // if no use found.
3409 if (!IsKillSet) {
3410 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
3411 // Use found, set it killed.
3412 IsKillSet = true;
3413 MO->setIsKill(true);
3414 continue;
3415 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
3416 &getRegisterInfo()))) {
3417 // No use found, set dead for its def.
3418 assert(&*It == StartMI && "No new def between StartMI and EndMI.");
3419 MO->setIsDead(true);
3420 break;
3421 }
3422 }
3423
3424 if ((&*It) == StartMI)
3425 break;
3426 }
3427 // Ensure RegMo liveness is killed after EndMI.
3428 assert((IsKillSet || (MO && MO->isDead())) &&
3429 "RegNo should be killed or dead");
3430 }
3431
3432 // This opt tries to convert the following imm form to an index form to save an
3433 // add for stack variables.
3434 // Return false if no such pattern found.
3435 //
3436 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3437 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3438 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3439 //
3440 // can be converted to:
3441 //
3442 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3443 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3444 //
3445 // In order to eliminate ADD instr, make sure that:
3446 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3447 // new ADDI instr and ADDI can only take int16 Imm.
3448 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3449 // between ADDI and ADD instr since its original def in ADDI will be changed
3450 // in new ADDI instr. And also there should be no new def for it between
3451 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3452 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3453 // between ADD and Imm instr since ADD instr will be eliminated.
3454 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3455 // moved to Index instr.
foldFrameOffset(MachineInstr & MI) const3456 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
3457 MachineFunction *MF = MI.getParent()->getParent();
3458 MachineRegisterInfo *MRI = &MF->getRegInfo();
3459 bool PostRA = !MRI->isSSA();
3460 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3461 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3462 // frame base(OffsetAddi) are determined.
3463 if (!PostRA)
3464 return false;
3465 unsigned ToBeDeletedReg = 0;
3466 int64_t OffsetImm = 0;
3467 unsigned XFormOpcode = 0;
3468 ImmInstrInfo III;
3469
3470 // Check if Imm instr meets requirement.
3471 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3472 III))
3473 return false;
3474
3475 bool OtherIntermediateUse = false;
3476 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3477
3478 // Exit if there is other use between ADD and Imm instr or no def found.
3479 if (OtherIntermediateUse || !ADDMI)
3480 return false;
3481
3482 // Check if ADD instr meets requirement.
3483 if (!isADDInstrEligibleForFolding(*ADDMI))
3484 return false;
3485
3486 unsigned ScaleRegIdx = 0;
3487 int64_t OffsetAddi = 0;
3488 MachineInstr *ADDIMI = nullptr;
3489
3490 // Check if there is a valid ToBeChangedReg in ADDMI.
3491 // 1: It must be killed.
3492 // 2: Its definition must be a valid ADDIMI.
3493 // 3: It must satify int16 offset requirement.
3494 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3495 ScaleRegIdx = 2;
3496 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3497 ScaleRegIdx = 1;
3498 else
3499 return false;
3500
3501 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3502 unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3503 unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3504 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3505 MachineBasicBlock::iterator End) {
3506 for (auto It = ++Start; It != End; It++)
3507 if (It->modifiesRegister(Reg, &getRegisterInfo()))
3508 return true;
3509 return false;
3510 };
3511
3512 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3513 // treated as special zero when ScaleReg is R0/X0 register.
3514 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3515 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3516 return false;
3517
3518 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3519 // and Imm Instr.
3520 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3521 return false;
3522
3523 // Now start to do the transformation.
3524 LLVM_DEBUG(dbgs() << "Replace instruction: "
3525 << "\n");
3526 LLVM_DEBUG(ADDIMI->dump());
3527 LLVM_DEBUG(ADDMI->dump());
3528 LLVM_DEBUG(MI.dump());
3529 LLVM_DEBUG(dbgs() << "with: "
3530 << "\n");
3531
3532 // Update ADDI instr.
3533 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3534
3535 // Update Imm instr.
3536 MI.setDesc(get(XFormOpcode));
3537 MI.getOperand(III.ImmOpNo)
3538 .ChangeToRegister(ScaleReg, false, false,
3539 ADDMI->getOperand(ScaleRegIdx).isKill());
3540
3541 MI.getOperand(III.OpNoForForwarding)
3542 .ChangeToRegister(ToBeChangedReg, false, false, true);
3543
3544 // Eliminate ADD instr.
3545 ADDMI->eraseFromParent();
3546
3547 LLVM_DEBUG(ADDIMI->dump());
3548 LLVM_DEBUG(MI.dump());
3549
3550 return true;
3551 }
3552
isADDIInstrEligibleForFolding(MachineInstr & ADDIMI,int64_t & Imm) const3553 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
3554 int64_t &Imm) const {
3555 unsigned Opc = ADDIMI.getOpcode();
3556
3557 // Exit if the instruction is not ADDI.
3558 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3559 return false;
3560
3561 // The operand may not necessarily be an immediate - it could be a relocation.
3562 if (!ADDIMI.getOperand(2).isImm())
3563 return false;
3564
3565 Imm = ADDIMI.getOperand(2).getImm();
3566
3567 return true;
3568 }
3569
isADDInstrEligibleForFolding(MachineInstr & ADDMI) const3570 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
3571 unsigned Opc = ADDMI.getOpcode();
3572
3573 // Exit if the instruction is not ADD.
3574 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3575 }
3576
isImmInstrEligibleForFolding(MachineInstr & MI,unsigned & ToBeDeletedReg,unsigned & XFormOpcode,int64_t & OffsetImm,ImmInstrInfo & III) const3577 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
3578 unsigned &ToBeDeletedReg,
3579 unsigned &XFormOpcode,
3580 int64_t &OffsetImm,
3581 ImmInstrInfo &III) const {
3582 // Only handle load/store.
3583 if (!MI.mayLoadOrStore())
3584 return false;
3585
3586 unsigned Opc = MI.getOpcode();
3587
3588 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3589
3590 // Exit if instruction has no index form.
3591 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3592 return false;
3593
3594 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3595 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
3596 III, true))
3597 return false;
3598
3599 if (!III.IsSummingOperands)
3600 return false;
3601
3602 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3603 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3604 // Only support imm operands, not relocation slots or others.
3605 if (!ImmOperand.isImm())
3606 return false;
3607
3608 assert(RegOperand.isReg() && "Instruction format is not right");
3609
3610 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3611 if (!RegOperand.isKill())
3612 return false;
3613
3614 ToBeDeletedReg = RegOperand.getReg();
3615 OffsetImm = ImmOperand.getImm();
3616
3617 return true;
3618 }
3619
isValidToBeChangedReg(MachineInstr * ADDMI,unsigned Index,MachineInstr * & ADDIMI,int64_t & OffsetAddi,int64_t OffsetImm) const3620 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
3621 MachineInstr *&ADDIMI,
3622 int64_t &OffsetAddi,
3623 int64_t OffsetImm) const {
3624 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3625 MachineOperand &MO = ADDMI->getOperand(Index);
3626
3627 if (!MO.isKill())
3628 return false;
3629
3630 bool OtherIntermediateUse = false;
3631
3632 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3633 // Currently handle only one "add + Imminstr" pair case, exit if other
3634 // intermediate use for ToBeChangedReg found.
3635 // TODO: handle the cases where there are other "add + Imminstr" pairs
3636 // with same offset in Imminstr which is like:
3637 //
3638 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3639 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3640 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3641 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3642 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3643 //
3644 // can be converted to:
3645 //
3646 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3647 // (OffsetAddi + OffsetImm)
3648 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3649 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3650
3651 if (OtherIntermediateUse || !ADDIMI)
3652 return false;
3653 // Check if ADDI instr meets requirement.
3654 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3655 return false;
3656
3657 if (isInt<16>(OffsetAddi + OffsetImm))
3658 return true;
3659 return false;
3660 }
3661
3662 // If this instruction has an immediate form and one of its operands is a
3663 // result of a load-immediate or an add-immediate, convert it to
3664 // the immediate form if the constant is in range.
convertToImmediateForm(MachineInstr & MI,MachineInstr ** KilledDef) const3665 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
3666 MachineInstr **KilledDef) const {
3667 MachineFunction *MF = MI.getParent()->getParent();
3668 MachineRegisterInfo *MRI = &MF->getRegInfo();
3669 bool PostRA = !MRI->isSSA();
3670 bool SeenIntermediateUse = true;
3671 unsigned ForwardingOperand = ~0U;
3672 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3673 SeenIntermediateUse);
3674 if (!DefMI)
3675 return false;
3676 assert(ForwardingOperand < MI.getNumOperands() &&
3677 "The forwarding operand needs to be valid at this point");
3678 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3679 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3680 if (KilledDef && KillFwdDefMI)
3681 *KilledDef = DefMI;
3682
3683 // If this is a imm instruction and its register operands is produced by ADDI,
3684 // put the imm into imm inst directly.
3685 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3686 PPC::INSTRUCTION_LIST_END &&
3687 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3688 return true;
3689
3690 ImmInstrInfo III;
3691 bool IsVFReg = MI.getOperand(0).isReg()
3692 ? isVFRegister(MI.getOperand(0).getReg())
3693 : false;
3694 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3695 // If this is a reg+reg instruction that has a reg+imm form,
3696 // and one of the operands is produced by an add-immediate,
3697 // try to convert it.
3698 if (HasImmForm &&
3699 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3700 KillFwdDefMI))
3701 return true;
3702
3703 // If this is a reg+reg instruction that has a reg+imm form,
3704 // and one of the operands is produced by LI, convert it now.
3705 if (HasImmForm &&
3706 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3707 return true;
3708
3709 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3710 // can be simpified to LI.
3711 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3712 return true;
3713
3714 return false;
3715 }
3716
combineRLWINM(MachineInstr & MI,MachineInstr ** ToErase) const3717 bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
3718 MachineInstr **ToErase) const {
3719 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3720 unsigned FoldingReg = MI.getOperand(1).getReg();
3721 if (!Register::isVirtualRegister(FoldingReg))
3722 return false;
3723 MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3724 if (SrcMI->getOpcode() != PPC::RLWINM &&
3725 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3726 SrcMI->getOpcode() != PPC::RLWINM8 &&
3727 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3728 return false;
3729 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3730 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3731 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3732 "Invalid PPC::RLWINM Instruction!");
3733 uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3734 uint64_t SHMI = MI.getOperand(2).getImm();
3735 uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3736 uint64_t MBMI = MI.getOperand(3).getImm();
3737 uint64_t MESrc = SrcMI->getOperand(4).getImm();
3738 uint64_t MEMI = MI.getOperand(4).getImm();
3739
3740 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3741 "Invalid PPC::RLWINM Instruction!");
3742 // If MBMI is bigger than MEMI, we always can not get run of ones.
3743 // RotatedSrcMask non-wrap:
3744 // 0........31|32........63
3745 // RotatedSrcMask: B---E B---E
3746 // MaskMI: -----------|--E B------
3747 // Result: ----- --- (Bad candidate)
3748 //
3749 // RotatedSrcMask wrap:
3750 // 0........31|32........63
3751 // RotatedSrcMask: --E B----|--E B----
3752 // MaskMI: -----------|--E B------
3753 // Result: --- -----|--- ----- (Bad candidate)
3754 //
3755 // One special case is RotatedSrcMask is a full set mask.
3756 // RotatedSrcMask full:
3757 // 0........31|32........63
3758 // RotatedSrcMask: ------EB---|-------EB---
3759 // MaskMI: -----------|--E B------
3760 // Result: -----------|--- ------- (Good candidate)
3761
3762 // Mark special case.
3763 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3764
3765 // For other MBMI > MEMI cases, just return.
3766 if ((MBMI > MEMI) && !SrcMaskFull)
3767 return false;
3768
3769 // Handle MBMI <= MEMI cases.
3770 APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3771 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3772 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3773 // while in PowerPC ISA, lowerest bit is at index 63.
3774 APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3775
3776 APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3777 APInt FinalMask = RotatedSrcMask & MaskMI;
3778 uint32_t NewMB, NewME;
3779 bool Simplified = false;
3780
3781 // If final mask is 0, MI result should be 0 too.
3782 if (FinalMask.isZero()) {
3783 bool Is64Bit =
3784 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3785 Simplified = true;
3786 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3787 LLVM_DEBUG(MI.dump());
3788
3789 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3790 // Replace MI with "LI 0"
3791 MI.RemoveOperand(4);
3792 MI.RemoveOperand(3);
3793 MI.RemoveOperand(2);
3794 MI.getOperand(1).ChangeToImmediate(0);
3795 MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3796 } else {
3797 // Replace MI with "ANDI_rec reg, 0"
3798 MI.RemoveOperand(4);
3799 MI.RemoveOperand(3);
3800 MI.getOperand(2).setImm(0);
3801 MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3802 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3803 if (SrcMI->getOperand(1).isKill()) {
3804 MI.getOperand(1).setIsKill(true);
3805 SrcMI->getOperand(1).setIsKill(false);
3806 } else
3807 // About to replace MI.getOperand(1), clear its kill flag.
3808 MI.getOperand(1).setIsKill(false);
3809 }
3810
3811 LLVM_DEBUG(dbgs() << "With: ");
3812 LLVM_DEBUG(MI.dump());
3813
3814 } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3815 NewMB <= NewME) ||
3816 SrcMaskFull) {
3817 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3818 // than NewME. Otherwise we get a 64 bit value after folding, but MI
3819 // return a 32 bit value.
3820 Simplified = true;
3821 LLVM_DEBUG(dbgs() << "Converting Instr: ");
3822 LLVM_DEBUG(MI.dump());
3823
3824 uint16_t NewSH = (SHSrc + SHMI) % 32;
3825 MI.getOperand(2).setImm(NewSH);
3826 // If SrcMI mask is full, no need to update MBMI and MEMI.
3827 if (!SrcMaskFull) {
3828 MI.getOperand(3).setImm(NewMB);
3829 MI.getOperand(4).setImm(NewME);
3830 }
3831 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3832 if (SrcMI->getOperand(1).isKill()) {
3833 MI.getOperand(1).setIsKill(true);
3834 SrcMI->getOperand(1).setIsKill(false);
3835 } else
3836 // About to replace MI.getOperand(1), clear its kill flag.
3837 MI.getOperand(1).setIsKill(false);
3838
3839 LLVM_DEBUG(dbgs() << "To: ");
3840 LLVM_DEBUG(MI.dump());
3841 }
3842 if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3843 !SrcMI->hasImplicitDef()) {
3844 // If FoldingReg has no non-debug use and it has no implicit def (it
3845 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3846 // Otherwise keep it.
3847 *ToErase = SrcMI;
3848 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3849 LLVM_DEBUG(SrcMI->dump());
3850 }
3851 return Simplified;
3852 }
3853
instrHasImmForm(unsigned Opc,bool IsVFReg,ImmInstrInfo & III,bool PostRA) const3854 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3855 ImmInstrInfo &III, bool PostRA) const {
3856 // The vast majority of the instructions would need their operand 2 replaced
3857 // with an immediate when switching to the reg+imm form. A marked exception
3858 // are the update form loads/stores for which a constant operand 2 would need
3859 // to turn into a displacement and move operand 1 to the operand 2 position.
3860 III.ImmOpNo = 2;
3861 III.OpNoForForwarding = 2;
3862 III.ImmWidth = 16;
3863 III.ImmMustBeMultipleOf = 1;
3864 III.TruncateImmTo = 0;
3865 III.IsSummingOperands = false;
3866 switch (Opc) {
3867 default: return false;
3868 case PPC::ADD4:
3869 case PPC::ADD8:
3870 III.SignedImm = true;
3871 III.ZeroIsSpecialOrig = 0;
3872 III.ZeroIsSpecialNew = 1;
3873 III.IsCommutative = true;
3874 III.IsSummingOperands = true;
3875 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3876 break;
3877 case PPC::ADDC:
3878 case PPC::ADDC8:
3879 III.SignedImm = true;
3880 III.ZeroIsSpecialOrig = 0;
3881 III.ZeroIsSpecialNew = 0;
3882 III.IsCommutative = true;
3883 III.IsSummingOperands = true;
3884 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3885 break;
3886 case PPC::ADDC_rec:
3887 III.SignedImm = true;
3888 III.ZeroIsSpecialOrig = 0;
3889 III.ZeroIsSpecialNew = 0;
3890 III.IsCommutative = true;
3891 III.IsSummingOperands = true;
3892 III.ImmOpcode = PPC::ADDIC_rec;
3893 break;
3894 case PPC::SUBFC:
3895 case PPC::SUBFC8:
3896 III.SignedImm = true;
3897 III.ZeroIsSpecialOrig = 0;
3898 III.ZeroIsSpecialNew = 0;
3899 III.IsCommutative = false;
3900 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3901 break;
3902 case PPC::CMPW:
3903 case PPC::CMPD:
3904 III.SignedImm = true;
3905 III.ZeroIsSpecialOrig = 0;
3906 III.ZeroIsSpecialNew = 0;
3907 III.IsCommutative = false;
3908 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3909 break;
3910 case PPC::CMPLW:
3911 case PPC::CMPLD:
3912 III.SignedImm = false;
3913 III.ZeroIsSpecialOrig = 0;
3914 III.ZeroIsSpecialNew = 0;
3915 III.IsCommutative = false;
3916 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3917 break;
3918 case PPC::AND_rec:
3919 case PPC::AND8_rec:
3920 case PPC::OR:
3921 case PPC::OR8:
3922 case PPC::XOR:
3923 case PPC::XOR8:
3924 III.SignedImm = false;
3925 III.ZeroIsSpecialOrig = 0;
3926 III.ZeroIsSpecialNew = 0;
3927 III.IsCommutative = true;
3928 switch(Opc) {
3929 default: llvm_unreachable("Unknown opcode");
3930 case PPC::AND_rec:
3931 III.ImmOpcode = PPC::ANDI_rec;
3932 break;
3933 case PPC::AND8_rec:
3934 III.ImmOpcode = PPC::ANDI8_rec;
3935 break;
3936 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3937 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3938 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3939 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3940 }
3941 break;
3942 case PPC::RLWNM:
3943 case PPC::RLWNM8:
3944 case PPC::RLWNM_rec:
3945 case PPC::RLWNM8_rec:
3946 case PPC::SLW:
3947 case PPC::SLW8:
3948 case PPC::SLW_rec:
3949 case PPC::SLW8_rec:
3950 case PPC::SRW:
3951 case PPC::SRW8:
3952 case PPC::SRW_rec:
3953 case PPC::SRW8_rec:
3954 case PPC::SRAW:
3955 case PPC::SRAW_rec:
3956 III.SignedImm = false;
3957 III.ZeroIsSpecialOrig = 0;
3958 III.ZeroIsSpecialNew = 0;
3959 III.IsCommutative = false;
3960 // This isn't actually true, but the instructions ignore any of the
3961 // upper bits, so any immediate loaded with an LI is acceptable.
3962 // This does not apply to shift right algebraic because a value
3963 // out of range will produce a -1/0.
3964 III.ImmWidth = 16;
3965 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3966 Opc == PPC::RLWNM8_rec)
3967 III.TruncateImmTo = 5;
3968 else
3969 III.TruncateImmTo = 6;
3970 switch(Opc) {
3971 default: llvm_unreachable("Unknown opcode");
3972 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3973 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3974 case PPC::RLWNM_rec:
3975 III.ImmOpcode = PPC::RLWINM_rec;
3976 break;
3977 case PPC::RLWNM8_rec:
3978 III.ImmOpcode = PPC::RLWINM8_rec;
3979 break;
3980 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
3981 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
3982 case PPC::SLW_rec:
3983 III.ImmOpcode = PPC::RLWINM_rec;
3984 break;
3985 case PPC::SLW8_rec:
3986 III.ImmOpcode = PPC::RLWINM8_rec;
3987 break;
3988 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
3989 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
3990 case PPC::SRW_rec:
3991 III.ImmOpcode = PPC::RLWINM_rec;
3992 break;
3993 case PPC::SRW8_rec:
3994 III.ImmOpcode = PPC::RLWINM8_rec;
3995 break;
3996 case PPC::SRAW:
3997 III.ImmWidth = 5;
3998 III.TruncateImmTo = 0;
3999 III.ImmOpcode = PPC::SRAWI;
4000 break;
4001 case PPC::SRAW_rec:
4002 III.ImmWidth = 5;
4003 III.TruncateImmTo = 0;
4004 III.ImmOpcode = PPC::SRAWI_rec;
4005 break;
4006 }
4007 break;
4008 case PPC::RLDCL:
4009 case PPC::RLDCL_rec:
4010 case PPC::RLDCR:
4011 case PPC::RLDCR_rec:
4012 case PPC::SLD:
4013 case PPC::SLD_rec:
4014 case PPC::SRD:
4015 case PPC::SRD_rec:
4016 case PPC::SRAD:
4017 case PPC::SRAD_rec:
4018 III.SignedImm = false;
4019 III.ZeroIsSpecialOrig = 0;
4020 III.ZeroIsSpecialNew = 0;
4021 III.IsCommutative = false;
4022 // This isn't actually true, but the instructions ignore any of the
4023 // upper bits, so any immediate loaded with an LI is acceptable.
4024 // This does not apply to shift right algebraic because a value
4025 // out of range will produce a -1/0.
4026 III.ImmWidth = 16;
4027 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4028 Opc == PPC::RLDCR_rec)
4029 III.TruncateImmTo = 6;
4030 else
4031 III.TruncateImmTo = 7;
4032 switch(Opc) {
4033 default: llvm_unreachable("Unknown opcode");
4034 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4035 case PPC::RLDCL_rec:
4036 III.ImmOpcode = PPC::RLDICL_rec;
4037 break;
4038 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4039 case PPC::RLDCR_rec:
4040 III.ImmOpcode = PPC::RLDICR_rec;
4041 break;
4042 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4043 case PPC::SLD_rec:
4044 III.ImmOpcode = PPC::RLDICR_rec;
4045 break;
4046 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4047 case PPC::SRD_rec:
4048 III.ImmOpcode = PPC::RLDICL_rec;
4049 break;
4050 case PPC::SRAD:
4051 III.ImmWidth = 6;
4052 III.TruncateImmTo = 0;
4053 III.ImmOpcode = PPC::SRADI;
4054 break;
4055 case PPC::SRAD_rec:
4056 III.ImmWidth = 6;
4057 III.TruncateImmTo = 0;
4058 III.ImmOpcode = PPC::SRADI_rec;
4059 break;
4060 }
4061 break;
4062 // Loads and stores:
4063 case PPC::LBZX:
4064 case PPC::LBZX8:
4065 case PPC::LHZX:
4066 case PPC::LHZX8:
4067 case PPC::LHAX:
4068 case PPC::LHAX8:
4069 case PPC::LWZX:
4070 case PPC::LWZX8:
4071 case PPC::LWAX:
4072 case PPC::LDX:
4073 case PPC::LFSX:
4074 case PPC::LFDX:
4075 case PPC::STBX:
4076 case PPC::STBX8:
4077 case PPC::STHX:
4078 case PPC::STHX8:
4079 case PPC::STWX:
4080 case PPC::STWX8:
4081 case PPC::STDX:
4082 case PPC::STFSX:
4083 case PPC::STFDX:
4084 III.SignedImm = true;
4085 III.ZeroIsSpecialOrig = 1;
4086 III.ZeroIsSpecialNew = 2;
4087 III.IsCommutative = true;
4088 III.IsSummingOperands = true;
4089 III.ImmOpNo = 1;
4090 III.OpNoForForwarding = 2;
4091 switch(Opc) {
4092 default: llvm_unreachable("Unknown opcode");
4093 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4094 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4095 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4096 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4097 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4098 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4099 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4100 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4101 case PPC::LWAX:
4102 III.ImmOpcode = PPC::LWA;
4103 III.ImmMustBeMultipleOf = 4;
4104 break;
4105 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4106 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4107 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4108 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4109 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4110 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4111 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4112 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4113 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4114 case PPC::STDX:
4115 III.ImmOpcode = PPC::STD;
4116 III.ImmMustBeMultipleOf = 4;
4117 break;
4118 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4119 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4120 }
4121 break;
4122 case PPC::LBZUX:
4123 case PPC::LBZUX8:
4124 case PPC::LHZUX:
4125 case PPC::LHZUX8:
4126 case PPC::LHAUX:
4127 case PPC::LHAUX8:
4128 case PPC::LWZUX:
4129 case PPC::LWZUX8:
4130 case PPC::LDUX:
4131 case PPC::LFSUX:
4132 case PPC::LFDUX:
4133 case PPC::STBUX:
4134 case PPC::STBUX8:
4135 case PPC::STHUX:
4136 case PPC::STHUX8:
4137 case PPC::STWUX:
4138 case PPC::STWUX8:
4139 case PPC::STDUX:
4140 case PPC::STFSUX:
4141 case PPC::STFDUX:
4142 III.SignedImm = true;
4143 III.ZeroIsSpecialOrig = 2;
4144 III.ZeroIsSpecialNew = 3;
4145 III.IsCommutative = false;
4146 III.IsSummingOperands = true;
4147 III.ImmOpNo = 2;
4148 III.OpNoForForwarding = 3;
4149 switch(Opc) {
4150 default: llvm_unreachable("Unknown opcode");
4151 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4152 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4153 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4154 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4155 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4156 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4157 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4158 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4159 case PPC::LDUX:
4160 III.ImmOpcode = PPC::LDU;
4161 III.ImmMustBeMultipleOf = 4;
4162 break;
4163 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4164 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4165 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4166 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4167 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4168 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4169 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4170 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4171 case PPC::STDUX:
4172 III.ImmOpcode = PPC::STDU;
4173 III.ImmMustBeMultipleOf = 4;
4174 break;
4175 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4176 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4177 }
4178 break;
4179 // Power9 and up only. For some of these, the X-Form version has access to all
4180 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4181 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4182 // into or stored from is one of the VR registers.
4183 case PPC::LXVX:
4184 case PPC::LXSSPX:
4185 case PPC::LXSDX:
4186 case PPC::STXVX:
4187 case PPC::STXSSPX:
4188 case PPC::STXSDX:
4189 case PPC::XFLOADf32:
4190 case PPC::XFLOADf64:
4191 case PPC::XFSTOREf32:
4192 case PPC::XFSTOREf64:
4193 if (!Subtarget.hasP9Vector())
4194 return false;
4195 III.SignedImm = true;
4196 III.ZeroIsSpecialOrig = 1;
4197 III.ZeroIsSpecialNew = 2;
4198 III.IsCommutative = true;
4199 III.IsSummingOperands = true;
4200 III.ImmOpNo = 1;
4201 III.OpNoForForwarding = 2;
4202 III.ImmMustBeMultipleOf = 4;
4203 switch(Opc) {
4204 default: llvm_unreachable("Unknown opcode");
4205 case PPC::LXVX:
4206 III.ImmOpcode = PPC::LXV;
4207 III.ImmMustBeMultipleOf = 16;
4208 break;
4209 case PPC::LXSSPX:
4210 if (PostRA) {
4211 if (IsVFReg)
4212 III.ImmOpcode = PPC::LXSSP;
4213 else {
4214 III.ImmOpcode = PPC::LFS;
4215 III.ImmMustBeMultipleOf = 1;
4216 }
4217 break;
4218 }
4219 LLVM_FALLTHROUGH;
4220 case PPC::XFLOADf32:
4221 III.ImmOpcode = PPC::DFLOADf32;
4222 break;
4223 case PPC::LXSDX:
4224 if (PostRA) {
4225 if (IsVFReg)
4226 III.ImmOpcode = PPC::LXSD;
4227 else {
4228 III.ImmOpcode = PPC::LFD;
4229 III.ImmMustBeMultipleOf = 1;
4230 }
4231 break;
4232 }
4233 LLVM_FALLTHROUGH;
4234 case PPC::XFLOADf64:
4235 III.ImmOpcode = PPC::DFLOADf64;
4236 break;
4237 case PPC::STXVX:
4238 III.ImmOpcode = PPC::STXV;
4239 III.ImmMustBeMultipleOf = 16;
4240 break;
4241 case PPC::STXSSPX:
4242 if (PostRA) {
4243 if (IsVFReg)
4244 III.ImmOpcode = PPC::STXSSP;
4245 else {
4246 III.ImmOpcode = PPC::STFS;
4247 III.ImmMustBeMultipleOf = 1;
4248 }
4249 break;
4250 }
4251 LLVM_FALLTHROUGH;
4252 case PPC::XFSTOREf32:
4253 III.ImmOpcode = PPC::DFSTOREf32;
4254 break;
4255 case PPC::STXSDX:
4256 if (PostRA) {
4257 if (IsVFReg)
4258 III.ImmOpcode = PPC::STXSD;
4259 else {
4260 III.ImmOpcode = PPC::STFD;
4261 III.ImmMustBeMultipleOf = 1;
4262 }
4263 break;
4264 }
4265 LLVM_FALLTHROUGH;
4266 case PPC::XFSTOREf64:
4267 III.ImmOpcode = PPC::DFSTOREf64;
4268 break;
4269 }
4270 break;
4271 }
4272 return true;
4273 }
4274
4275 // Utility function for swaping two arbitrary operands of an instruction.
swapMIOperands(MachineInstr & MI,unsigned Op1,unsigned Op2)4276 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4277 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4278
4279 unsigned MaxOp = std::max(Op1, Op2);
4280 unsigned MinOp = std::min(Op1, Op2);
4281 MachineOperand MOp1 = MI.getOperand(MinOp);
4282 MachineOperand MOp2 = MI.getOperand(MaxOp);
4283 MI.RemoveOperand(std::max(Op1, Op2));
4284 MI.RemoveOperand(std::min(Op1, Op2));
4285
4286 // If the operands we are swapping are the two at the end (the common case)
4287 // we can just remove both and add them in the opposite order.
4288 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4289 MI.addOperand(MOp2);
4290 MI.addOperand(MOp1);
4291 } else {
4292 // Store all operands in a temporary vector, remove them and re-add in the
4293 // right order.
4294 SmallVector<MachineOperand, 2> MOps;
4295 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4296 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4297 MOps.push_back(MI.getOperand(i));
4298 MI.RemoveOperand(i);
4299 }
4300 // MOp2 needs to be added next.
4301 MI.addOperand(MOp2);
4302 // Now add the rest.
4303 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4304 if (i == MaxOp)
4305 MI.addOperand(MOp1);
4306 else {
4307 MI.addOperand(MOps.back());
4308 MOps.pop_back();
4309 }
4310 }
4311 }
4312 }
4313
4314 // Check if the 'MI' that has the index OpNoForForwarding
4315 // meets the requirement described in the ImmInstrInfo.
isUseMIElgibleForForwarding(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding) const4316 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4317 const ImmInstrInfo &III,
4318 unsigned OpNoForForwarding
4319 ) const {
4320 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4321 // would not work pre-RA, we can only do the check post RA.
4322 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4323 if (MRI.isSSA())
4324 return false;
4325
4326 // Cannot do the transform if MI isn't summing the operands.
4327 if (!III.IsSummingOperands)
4328 return false;
4329
4330 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4331 if (!III.ZeroIsSpecialOrig)
4332 return false;
4333
4334 // We cannot do the transform if the operand we are trying to replace
4335 // isn't the same as the operand the instruction allows.
4336 if (OpNoForForwarding != III.OpNoForForwarding)
4337 return false;
4338
4339 // Check if the instruction we are trying to transform really has
4340 // the special zero register as its operand.
4341 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4342 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4343 return false;
4344
4345 // This machine instruction is convertible if it is,
4346 // 1. summing the operands.
4347 // 2. one of the operands is special zero register.
4348 // 3. the operand we are trying to replace is allowed by the MI.
4349 return true;
4350 }
4351
4352 // Check if the DefMI is the add inst and set the ImmMO and RegMO
4353 // accordingly.
isDefMIElgibleForForwarding(MachineInstr & DefMI,const ImmInstrInfo & III,MachineOperand * & ImmMO,MachineOperand * & RegMO) const4354 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4355 const ImmInstrInfo &III,
4356 MachineOperand *&ImmMO,
4357 MachineOperand *&RegMO) const {
4358 unsigned Opc = DefMI.getOpcode();
4359 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4360 return false;
4361
4362 assert(DefMI.getNumOperands() >= 3 &&
4363 "Add inst must have at least three operands");
4364 RegMO = &DefMI.getOperand(1);
4365 ImmMO = &DefMI.getOperand(2);
4366
4367 // Before RA, ADDI first operand could be a frame index.
4368 if (!RegMO->isReg())
4369 return false;
4370
4371 // This DefMI is elgible for forwarding if it is:
4372 // 1. add inst
4373 // 2. one of the operands is Imm/CPI/Global.
4374 return isAnImmediateOperand(*ImmMO);
4375 }
4376
isRegElgibleForForwarding(const MachineOperand & RegMO,const MachineInstr & DefMI,const MachineInstr & MI,bool KillDefMI,bool & IsFwdFeederRegKilled) const4377 bool PPCInstrInfo::isRegElgibleForForwarding(
4378 const MachineOperand &RegMO, const MachineInstr &DefMI,
4379 const MachineInstr &MI, bool KillDefMI,
4380 bool &IsFwdFeederRegKilled) const {
4381 // x = addi y, imm
4382 // ...
4383 // z = lfdx 0, x -> z = lfd imm(y)
4384 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4385 // of "y" between the DEF of "x" and "z".
4386 // The query is only valid post RA.
4387 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4388 if (MRI.isSSA())
4389 return false;
4390
4391 Register Reg = RegMO.getReg();
4392
4393 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4394 MachineBasicBlock::const_reverse_iterator It = MI;
4395 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4396 It++;
4397 for (; It != E; ++It) {
4398 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4399 return false;
4400 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4401 IsFwdFeederRegKilled = true;
4402 // Made it to DefMI without encountering a clobber.
4403 if ((&*It) == &DefMI)
4404 break;
4405 }
4406 assert((&*It) == &DefMI && "DefMI is missing");
4407
4408 // If DefMI also defines the register to be forwarded, we can only forward it
4409 // if DefMI is being erased.
4410 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4411 return KillDefMI;
4412
4413 return true;
4414 }
4415
isImmElgibleForForwarding(const MachineOperand & ImmMO,const MachineInstr & DefMI,const ImmInstrInfo & III,int64_t & Imm,int64_t BaseImm) const4416 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4417 const MachineInstr &DefMI,
4418 const ImmInstrInfo &III,
4419 int64_t &Imm,
4420 int64_t BaseImm) const {
4421 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4422 if (DefMI.getOpcode() == PPC::ADDItocL) {
4423 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4424 // However, we know that, it is 16-bit width, and has the alignment of 4.
4425 // Check if the instruction met the requirement.
4426 if (III.ImmMustBeMultipleOf > 4 ||
4427 III.TruncateImmTo || III.ImmWidth != 16)
4428 return false;
4429
4430 // Going from XForm to DForm loads means that the displacement needs to be
4431 // not just an immediate but also a multiple of 4, or 16 depending on the
4432 // load. A DForm load cannot be represented if it is a multiple of say 2.
4433 // XForm loads do not have this restriction.
4434 if (ImmMO.isGlobal()) {
4435 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4436 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
4437 return false;
4438 }
4439
4440 return true;
4441 }
4442
4443 if (ImmMO.isImm()) {
4444 // It is Imm, we need to check if the Imm fit the range.
4445 // Sign-extend to 64-bits.
4446 // DefMI may be folded with another imm form instruction, the result Imm is
4447 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4448 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4449 if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4450 return false;
4451 if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4452 return false;
4453 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4454
4455 if (Imm % III.ImmMustBeMultipleOf)
4456 return false;
4457 if (III.TruncateImmTo)
4458 Imm &= ((1 << III.TruncateImmTo) - 1);
4459 }
4460 else
4461 return false;
4462
4463 // This ImmMO is forwarded if it meets the requriement describle
4464 // in ImmInstrInfo
4465 return true;
4466 }
4467
simplifyToLI(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding,MachineInstr ** KilledDef) const4468 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4469 unsigned OpNoForForwarding,
4470 MachineInstr **KilledDef) const {
4471 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4472 !DefMI.getOperand(1).isImm())
4473 return false;
4474
4475 MachineFunction *MF = MI.getParent()->getParent();
4476 MachineRegisterInfo *MRI = &MF->getRegInfo();
4477 bool PostRA = !MRI->isSSA();
4478
4479 int64_t Immediate = DefMI.getOperand(1).getImm();
4480 // Sign-extend to 64-bits.
4481 int64_t SExtImm = SignExtend64<16>(Immediate);
4482
4483 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
4484 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4485
4486 bool ReplaceWithLI = false;
4487 bool Is64BitLI = false;
4488 int64_t NewImm = 0;
4489 bool SetCR = false;
4490 unsigned Opc = MI.getOpcode();
4491 switch (Opc) {
4492 default:
4493 return false;
4494
4495 // FIXME: Any branches conditional on such a comparison can be made
4496 // unconditional. At this time, this happens too infrequently to be worth
4497 // the implementation effort, but if that ever changes, we could convert
4498 // such a pattern here.
4499 case PPC::CMPWI:
4500 case PPC::CMPLWI:
4501 case PPC::CMPDI:
4502 case PPC::CMPLDI: {
4503 // Doing this post-RA would require dataflow analysis to reliably find uses
4504 // of the CR register set by the compare.
4505 // No need to fixup killed/dead flag since this transformation is only valid
4506 // before RA.
4507 if (PostRA)
4508 return false;
4509 // If a compare-immediate is fed by an immediate and is itself an input of
4510 // an ISEL (the most common case) into a COPY of the correct register.
4511 bool Changed = false;
4512 Register DefReg = MI.getOperand(0).getReg();
4513 int64_t Comparand = MI.getOperand(2).getImm();
4514 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4515 ? (Comparand | 0xFFFFFFFFFFFF0000)
4516 : Comparand;
4517
4518 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4519 unsigned UseOpc = CompareUseMI.getOpcode();
4520 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4521 continue;
4522 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4523 Register TrueReg = CompareUseMI.getOperand(1).getReg();
4524 Register FalseReg = CompareUseMI.getOperand(2).getReg();
4525 unsigned RegToCopy =
4526 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4527 if (RegToCopy == PPC::NoRegister)
4528 continue;
4529 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4530 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4531 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4532 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4533 CompareUseMI.RemoveOperand(3);
4534 CompareUseMI.RemoveOperand(2);
4535 continue;
4536 }
4537 LLVM_DEBUG(
4538 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4539 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4540 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4541 // Convert to copy and remove unneeded operands.
4542 CompareUseMI.setDesc(get(PPC::COPY));
4543 CompareUseMI.RemoveOperand(3);
4544 CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1);
4545 CmpIselsConverted++;
4546 Changed = true;
4547 LLVM_DEBUG(CompareUseMI.dump());
4548 }
4549 if (Changed)
4550 return true;
4551 // This may end up incremented multiple times since this function is called
4552 // during a fixed-point transformation, but it is only meant to indicate the
4553 // presence of this opportunity.
4554 MissedConvertibleImmediateInstrs++;
4555 return false;
4556 }
4557
4558 // Immediate forms - may simply be convertable to an LI.
4559 case PPC::ADDI:
4560 case PPC::ADDI8: {
4561 // Does the sum fit in a 16-bit signed field?
4562 int64_t Addend = MI.getOperand(2).getImm();
4563 if (isInt<16>(Addend + SExtImm)) {
4564 ReplaceWithLI = true;
4565 Is64BitLI = Opc == PPC::ADDI8;
4566 NewImm = Addend + SExtImm;
4567 break;
4568 }
4569 return false;
4570 }
4571 case PPC::SUBFIC:
4572 case PPC::SUBFIC8: {
4573 // Only transform this if the CARRY implicit operand is dead.
4574 if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4575 return false;
4576 int64_t Minuend = MI.getOperand(2).getImm();
4577 if (isInt<16>(Minuend - SExtImm)) {
4578 ReplaceWithLI = true;
4579 Is64BitLI = Opc == PPC::SUBFIC8;
4580 NewImm = Minuend - SExtImm;
4581 break;
4582 }
4583 return false;
4584 }
4585 case PPC::RLDICL:
4586 case PPC::RLDICL_rec:
4587 case PPC::RLDICL_32:
4588 case PPC::RLDICL_32_64: {
4589 // Use APInt's rotate function.
4590 int64_t SH = MI.getOperand(2).getImm();
4591 int64_t MB = MI.getOperand(3).getImm();
4592 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4593 SExtImm, true);
4594 InVal = InVal.rotl(SH);
4595 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4596 InVal &= Mask;
4597 // Can't replace negative values with an LI as that will sign-extend
4598 // and not clear the left bits. If we're setting the CR bit, we will use
4599 // ANDI_rec which won't sign extend, so that's safe.
4600 if (isUInt<15>(InVal.getSExtValue()) ||
4601 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4602 ReplaceWithLI = true;
4603 Is64BitLI = Opc != PPC::RLDICL_32;
4604 NewImm = InVal.getSExtValue();
4605 SetCR = Opc == PPC::RLDICL_rec;
4606 break;
4607 }
4608 return false;
4609 }
4610 case PPC::RLWINM:
4611 case PPC::RLWINM8:
4612 case PPC::RLWINM_rec:
4613 case PPC::RLWINM8_rec: {
4614 int64_t SH = MI.getOperand(2).getImm();
4615 int64_t MB = MI.getOperand(3).getImm();
4616 int64_t ME = MI.getOperand(4).getImm();
4617 APInt InVal(32, SExtImm, true);
4618 InVal = InVal.rotl(SH);
4619 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4620 InVal &= Mask;
4621 // Can't replace negative values with an LI as that will sign-extend
4622 // and not clear the left bits. If we're setting the CR bit, we will use
4623 // ANDI_rec which won't sign extend, so that's safe.
4624 bool ValueFits = isUInt<15>(InVal.getSExtValue());
4625 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4626 isUInt<16>(InVal.getSExtValue()));
4627 if (ValueFits) {
4628 ReplaceWithLI = true;
4629 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4630 NewImm = InVal.getSExtValue();
4631 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4632 break;
4633 }
4634 return false;
4635 }
4636 case PPC::ORI:
4637 case PPC::ORI8:
4638 case PPC::XORI:
4639 case PPC::XORI8: {
4640 int64_t LogicalImm = MI.getOperand(2).getImm();
4641 int64_t Result = 0;
4642 if (Opc == PPC::ORI || Opc == PPC::ORI8)
4643 Result = LogicalImm | SExtImm;
4644 else
4645 Result = LogicalImm ^ SExtImm;
4646 if (isInt<16>(Result)) {
4647 ReplaceWithLI = true;
4648 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4649 NewImm = Result;
4650 break;
4651 }
4652 return false;
4653 }
4654 }
4655
4656 if (ReplaceWithLI) {
4657 // We need to be careful with CR-setting instructions we're replacing.
4658 if (SetCR) {
4659 // We don't know anything about uses when we're out of SSA, so only
4660 // replace if the new immediate will be reproduced.
4661 bool ImmChanged = (SExtImm & NewImm) != NewImm;
4662 if (PostRA && ImmChanged)
4663 return false;
4664
4665 if (!PostRA) {
4666 // If the defining load-immediate has no other uses, we can just replace
4667 // the immediate with the new immediate.
4668 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4669 DefMI.getOperand(1).setImm(NewImm);
4670
4671 // If we're not using the GPR result of the CR-setting instruction, we
4672 // just need to and with zero/non-zero depending on the new immediate.
4673 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
4674 if (NewImm) {
4675 assert(Immediate && "Transformation converted zero to non-zero?");
4676 NewImm = Immediate;
4677 }
4678 } else if (ImmChanged)
4679 return false;
4680 }
4681 }
4682
4683 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4684 LLVM_DEBUG(MI.dump());
4685 LLVM_DEBUG(dbgs() << "Fed by:\n");
4686 LLVM_DEBUG(DefMI.dump());
4687 LoadImmediateInfo LII;
4688 LII.Imm = NewImm;
4689 LII.Is64Bit = Is64BitLI;
4690 LII.SetCR = SetCR;
4691 // If we're setting the CR, the original load-immediate must be kept (as an
4692 // operand to ANDI_rec/ANDI8_rec).
4693 if (KilledDef && SetCR)
4694 *KilledDef = nullptr;
4695 replaceInstrWithLI(MI, LII);
4696
4697 // Fixup killed/dead flag after transformation.
4698 // Pattern:
4699 // ForwardingOperandReg = LI imm1
4700 // y = op2 imm2, ForwardingOperandReg(killed)
4701 if (IsForwardingOperandKilled)
4702 fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg);
4703
4704 LLVM_DEBUG(dbgs() << "With:\n");
4705 LLVM_DEBUG(MI.dump());
4706 return true;
4707 }
4708 return false;
4709 }
4710
transformToNewImmFormFedByAdd(MachineInstr & MI,MachineInstr & DefMI,unsigned OpNoForForwarding) const4711 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4712 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4713 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4714 bool PostRA = !MRI->isSSA();
4715 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4716 // for post-ra.
4717 if (PostRA)
4718 return false;
4719
4720 // Only handle load/store.
4721 if (!MI.mayLoadOrStore())
4722 return false;
4723
4724 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
4725
4726 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4727 "MI must have x-form opcode");
4728
4729 // get Imm Form info.
4730 ImmInstrInfo III;
4731 bool IsVFReg = MI.getOperand(0).isReg()
4732 ? isVFRegister(MI.getOperand(0).getReg())
4733 : false;
4734
4735 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
4736 return false;
4737
4738 if (!III.IsSummingOperands)
4739 return false;
4740
4741 if (OpNoForForwarding != III.OpNoForForwarding)
4742 return false;
4743
4744 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
4745 if (!ImmOperandMI.isImm())
4746 return false;
4747
4748 // Check DefMI.
4749 MachineOperand *ImmMO = nullptr;
4750 MachineOperand *RegMO = nullptr;
4751 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4752 return false;
4753 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4754
4755 // Check Imm.
4756 // Set ImmBase from imm instruction as base and get new Imm inside
4757 // isImmElgibleForForwarding.
4758 int64_t ImmBase = ImmOperandMI.getImm();
4759 int64_t Imm = 0;
4760 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
4761 return false;
4762
4763 // Get killed info in case fixup needed after transformation.
4764 unsigned ForwardKilledOperandReg = ~0U;
4765 if (MI.getOperand(III.OpNoForForwarding).isKill())
4766 ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
4767
4768 // Do the transform
4769 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4770 LLVM_DEBUG(MI.dump());
4771 LLVM_DEBUG(dbgs() << "Fed by:\n");
4772 LLVM_DEBUG(DefMI.dump());
4773
4774 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
4775 if (RegMO->isKill()) {
4776 MI.getOperand(III.OpNoForForwarding).setIsKill(true);
4777 // Clear the killed flag in RegMO. Doing this here can handle some cases
4778 // that DefMI and MI are not in same basic block.
4779 RegMO->setIsKill(false);
4780 }
4781 MI.getOperand(III.ImmOpNo).setImm(Imm);
4782
4783 // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
4784 if (DefMI.getParent() == MI.getParent()) {
4785 // Check if reg is killed between MI and DefMI.
4786 auto IsKilledFor = [&](unsigned Reg) {
4787 MachineBasicBlock::const_reverse_iterator It = MI;
4788 MachineBasicBlock::const_reverse_iterator E = DefMI;
4789 It++;
4790 for (; It != E; ++It) {
4791 if (It->killsRegister(Reg))
4792 return true;
4793 }
4794 return false;
4795 };
4796
4797 // Update kill flag
4798 if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
4799 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
4800 if (ForwardKilledOperandReg != ~0U)
4801 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4802 }
4803
4804 LLVM_DEBUG(dbgs() << "With:\n");
4805 LLVM_DEBUG(MI.dump());
4806 return true;
4807 }
4808
4809 // If an X-Form instruction is fed by an add-immediate and one of its operands
4810 // is the literal zero, attempt to forward the source of the add-immediate to
4811 // the corresponding D-Form instruction with the displacement coming from
4812 // the immediate being added.
transformToImmFormFedByAdd(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding,MachineInstr & DefMI,bool KillDefMI) const4813 bool PPCInstrInfo::transformToImmFormFedByAdd(
4814 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4815 MachineInstr &DefMI, bool KillDefMI) const {
4816 // RegMO ImmMO
4817 // | |
4818 // x = addi reg, imm <----- DefMI
4819 // y = op 0 , x <----- MI
4820 // |
4821 // OpNoForForwarding
4822 // Check if the MI meet the requirement described in the III.
4823 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
4824 return false;
4825
4826 // Check if the DefMI meet the requirement
4827 // described in the III. If yes, set the ImmMO and RegMO accordingly.
4828 MachineOperand *ImmMO = nullptr;
4829 MachineOperand *RegMO = nullptr;
4830 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4831 return false;
4832 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4833
4834 // As we get the Imm operand now, we need to check if the ImmMO meet
4835 // the requirement described in the III. If yes set the Imm.
4836 int64_t Imm = 0;
4837 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4838 return false;
4839
4840 bool IsFwdFeederRegKilled = false;
4841 // Check if the RegMO can be forwarded to MI.
4842 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4843 IsFwdFeederRegKilled))
4844 return false;
4845
4846 // Get killed info in case fixup needed after transformation.
4847 unsigned ForwardKilledOperandReg = ~0U;
4848 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4849 bool PostRA = !MRI.isSSA();
4850 if (PostRA && MI.getOperand(OpNoForForwarding).isKill())
4851 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4852
4853 // We know that, the MI and DefMI both meet the pattern, and
4854 // the Imm also meet the requirement with the new Imm-form.
4855 // It is safe to do the transformation now.
4856 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4857 LLVM_DEBUG(MI.dump());
4858 LLVM_DEBUG(dbgs() << "Fed by:\n");
4859 LLVM_DEBUG(DefMI.dump());
4860
4861 // Update the base reg first.
4862 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4863 false, false,
4864 RegMO->isKill());
4865
4866 // Then, update the imm.
4867 if (ImmMO->isImm()) {
4868 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4869 // directly.
4870 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4871 }
4872 else {
4873 // Otherwise, it is Constant Pool Index(CPI) or Global,
4874 // which is relocation in fact. We need to replace the special zero
4875 // register with ImmMO.
4876 // Before that, we need to fixup the target flags for imm.
4877 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4878 if (DefMI.getOpcode() == PPC::ADDItocL)
4879 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4880
4881 // MI didn't have the interface such as MI.setOperand(i) though
4882 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4883 // ImmMO, we need to remove ZERO operand and all the operands behind it,
4884 // and, add the ImmMO, then, move back all the operands behind ZERO.
4885 SmallVector<MachineOperand, 2> MOps;
4886 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4887 MOps.push_back(MI.getOperand(i));
4888 MI.RemoveOperand(i);
4889 }
4890
4891 // Remove the last MO in the list, which is ZERO operand in fact.
4892 MOps.pop_back();
4893 // Add the imm operand.
4894 MI.addOperand(*ImmMO);
4895 // Now add the rest back.
4896 for (auto &MO : MOps)
4897 MI.addOperand(MO);
4898 }
4899
4900 // Update the opcode.
4901 MI.setDesc(get(III.ImmOpcode));
4902
4903 // Fix up killed/dead flag after transformation.
4904 // Pattern 1:
4905 // x = ADD KilledFwdFeederReg, imm
4906 // n = opn KilledFwdFeederReg(killed), regn
4907 // y = XOP 0, x
4908 // Pattern 2:
4909 // x = ADD reg(killed), imm
4910 // y = XOP 0, x
4911 if (IsFwdFeederRegKilled || RegMO->isKill())
4912 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
4913 // Pattern 3:
4914 // ForwardKilledOperandReg = ADD reg, imm
4915 // y = XOP 0, ForwardKilledOperandReg(killed)
4916 if (ForwardKilledOperandReg != ~0U)
4917 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4918
4919 LLVM_DEBUG(dbgs() << "With:\n");
4920 LLVM_DEBUG(MI.dump());
4921
4922 return true;
4923 }
4924
transformToImmFormFedByLI(MachineInstr & MI,const ImmInstrInfo & III,unsigned ConstantOpNo,MachineInstr & DefMI) const4925 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4926 const ImmInstrInfo &III,
4927 unsigned ConstantOpNo,
4928 MachineInstr &DefMI) const {
4929 // DefMI must be LI or LI8.
4930 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4931 !DefMI.getOperand(1).isImm())
4932 return false;
4933
4934 // Get Imm operand and Sign-extend to 64-bits.
4935 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4936
4937 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4938 bool PostRA = !MRI.isSSA();
4939 // Exit early if we can't convert this.
4940 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4941 return false;
4942 if (Imm % III.ImmMustBeMultipleOf)
4943 return false;
4944 if (III.TruncateImmTo)
4945 Imm &= ((1 << III.TruncateImmTo) - 1);
4946 if (III.SignedImm) {
4947 APInt ActualValue(64, Imm, true);
4948 if (!ActualValue.isSignedIntN(III.ImmWidth))
4949 return false;
4950 } else {
4951 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4952 if ((uint64_t)Imm > UnsignedMax)
4953 return false;
4954 }
4955
4956 // If we're post-RA, the instructions don't agree on whether register zero is
4957 // special, we can transform this as long as the register operand that will
4958 // end up in the location where zero is special isn't R0.
4959 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4960 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4961 III.ZeroIsSpecialNew + 1;
4962 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4963 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4964 // If R0 is in the operand where zero is special for the new instruction,
4965 // it is unsafe to transform if the constant operand isn't that operand.
4966 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4967 ConstantOpNo != III.ZeroIsSpecialNew)
4968 return false;
4969 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4970 ConstantOpNo != PosForOrigZero)
4971 return false;
4972 }
4973
4974 // Get killed info in case fixup needed after transformation.
4975 unsigned ForwardKilledOperandReg = ~0U;
4976 if (PostRA && MI.getOperand(ConstantOpNo).isKill())
4977 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg();
4978
4979 unsigned Opc = MI.getOpcode();
4980 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4981 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4982 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4983 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4984 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4985 Opc == PPC::SRD || Opc == PPC::SRD_rec;
4986 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4987 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4988 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4989 Opc == PPC::SRD_rec;
4990
4991 MI.setDesc(get(III.ImmOpcode));
4992 if (ConstantOpNo == III.OpNoForForwarding) {
4993 // Converting shifts to immediate form is a bit tricky since they may do
4994 // one of three things:
4995 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4996 // 2. If the shift amount is zero, the result is unchanged (save for maybe
4997 // setting CR0)
4998 // 3. If the shift amount is in [1, OpSize), it's just a shift
4999 if (SpecialShift32 || SpecialShift64) {
5000 LoadImmediateInfo LII;
5001 LII.Imm = 0;
5002 LII.SetCR = SetCR;
5003 LII.Is64Bit = SpecialShift64;
5004 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
5005 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
5006 replaceInstrWithLI(MI, LII);
5007 // Shifts by zero don't change the value. If we don't need to set CR0,
5008 // just convert this to a COPY. Can't do this post-RA since we've already
5009 // cleaned up the copies.
5010 else if (!SetCR && ShAmt == 0 && !PostRA) {
5011 MI.RemoveOperand(2);
5012 MI.setDesc(get(PPC::COPY));
5013 } else {
5014 // The 32 bit and 64 bit instructions are quite different.
5015 if (SpecialShift32) {
5016 // Left shifts use (N, 0, 31-N).
5017 // Right shifts use (32-N, N, 31) if 0 < N < 32.
5018 // use (0, 0, 31) if N == 0.
5019 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
5020 uint64_t MB = RightShift ? ShAmt : 0;
5021 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
5022 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
5023 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
5024 .addImm(ME);
5025 } else {
5026 // Left shifts use (N, 63-N).
5027 // Right shifts use (64-N, N) if 0 < N < 64.
5028 // use (0, 0) if N == 0.
5029 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
5030 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
5031 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
5032 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
5033 }
5034 }
5035 } else
5036 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5037 }
5038 // Convert commutative instructions (switch the operands and convert the
5039 // desired one to an immediate.
5040 else if (III.IsCommutative) {
5041 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5042 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
5043 } else
5044 llvm_unreachable("Should have exited early!");
5045
5046 // For instructions for which the constant register replaces a different
5047 // operand than where the immediate goes, we need to swap them.
5048 if (III.OpNoForForwarding != III.ImmOpNo)
5049 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
5050
5051 // If the special R0/X0 register index are different for original instruction
5052 // and new instruction, we need to fix up the register class in new
5053 // instruction.
5054 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5055 if (III.ZeroIsSpecialNew) {
5056 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
5057 // need to fix up register class.
5058 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
5059 if (Register::isVirtualRegister(RegToModify)) {
5060 const TargetRegisterClass *NewRC =
5061 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
5062 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
5063 MRI.setRegClass(RegToModify, NewRC);
5064 }
5065 }
5066 }
5067
5068 // Fix up killed/dead flag after transformation.
5069 // Pattern:
5070 // ForwardKilledOperandReg = LI imm
5071 // y = XOP reg, ForwardKilledOperandReg(killed)
5072 if (ForwardKilledOperandReg != ~0U)
5073 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
5074 return true;
5075 }
5076
5077 const TargetRegisterClass *
updatedRC(const TargetRegisterClass * RC) const5078 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
5079 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
5080 return &PPC::VSRCRegClass;
5081 return RC;
5082 }
5083
getRecordFormOpcode(unsigned Opcode)5084 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
5085 return PPC::getRecordFormOpcode(Opcode);
5086 }
5087
5088 // This function returns true if the machine instruction
5089 // always outputs a value by sign-extending a 32 bit value,
5090 // i.e. 0 to 31-th bits are same as 32-th bit.
isSignExtendingOp(const MachineInstr & MI)5091 static bool isSignExtendingOp(const MachineInstr &MI) {
5092 int Opcode = MI.getOpcode();
5093 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS ||
5094 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec ||
5095 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA ||
5096 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 ||
5097 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 ||
5098 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX ||
5099 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU ||
5100 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
5101 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 ||
5102 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX ||
5103 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB ||
5104 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH ||
5105 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 ||
5106 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW ||
5107 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 ||
5108 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 ||
5109 Opcode == PPC::EXTSB8_32_64)
5110 return true;
5111
5112 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33)
5113 return true;
5114
5115 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5116 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
5117 MI.getOperand(3).getImm() > 0 &&
5118 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
5119 return true;
5120
5121 return false;
5122 }
5123
5124 // This function returns true if the machine instruction
5125 // always outputs zeros in higher 32 bits.
isZeroExtendingOp(const MachineInstr & MI)5126 static bool isZeroExtendingOp(const MachineInstr &MI) {
5127 int Opcode = MI.getOpcode();
5128 // The 16-bit immediate is sign-extended in li/lis.
5129 // If the most significant bit is zero, all higher bits are zero.
5130 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
5131 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
5132 int64_t Imm = MI.getOperand(1).getImm();
5133 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
5134 return true;
5135 }
5136
5137 // We have some variations of rotate-and-mask instructions
5138 // that clear higher 32-bits.
5139 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
5140 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
5141 Opcode == PPC::RLDICL_32_64) &&
5142 MI.getOperand(3).getImm() >= 32)
5143 return true;
5144
5145 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
5146 MI.getOperand(3).getImm() >= 32 &&
5147 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm())
5148 return true;
5149
5150 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5151 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
5152 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
5153 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
5154 return true;
5155
5156 // There are other instructions that clear higher 32-bits.
5157 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec ||
5158 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec ||
5159 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 ||
5160 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec ||
5161 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec ||
5162 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW ||
5163 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec ||
5164 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI ||
5165 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI ||
5166 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX ||
5167 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX ||
5168 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX ||
5169 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ ||
5170 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX ||
5171 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 ||
5172 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 ||
5173 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 ||
5174 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 ||
5175 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
5176 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec ||
5177 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec ||
5178 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec ||
5179 Opcode == PPC::MFVSRWZ)
5180 return true;
5181
5182 return false;
5183 }
5184
5185 // This function returns true if the input MachineInstr is a TOC save
5186 // instruction.
isTOCSaveMI(const MachineInstr & MI) const5187 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
5188 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
5189 return false;
5190 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5191 unsigned StackOffset = MI.getOperand(1).getImm();
5192 Register StackReg = MI.getOperand(2).getReg();
5193 Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
5194 if (StackReg == SPReg && StackOffset == TOCSaveOffset)
5195 return true;
5196
5197 return false;
5198 }
5199
5200 // We limit the max depth to track incoming values of PHIs or binary ops
5201 // (e.g. AND) to avoid excessive cost.
5202 const unsigned MAX_DEPTH = 1;
5203
5204 bool
isSignOrZeroExtended(const MachineInstr & MI,bool SignExt,const unsigned Depth) const5205 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
5206 const unsigned Depth) const {
5207 const MachineFunction *MF = MI.getParent()->getParent();
5208 const MachineRegisterInfo *MRI = &MF->getRegInfo();
5209
5210 // If we know this instruction returns sign- or zero-extended result,
5211 // return true.
5212 if (SignExt ? isSignExtendingOp(MI):
5213 isZeroExtendingOp(MI))
5214 return true;
5215
5216 switch (MI.getOpcode()) {
5217 case PPC::COPY: {
5218 Register SrcReg = MI.getOperand(1).getReg();
5219
5220 // In both ELFv1 and v2 ABI, method parameters and the return value
5221 // are sign- or zero-extended.
5222 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5223 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
5224 // We check the ZExt/SExt flags for a method parameter.
5225 if (MI.getParent()->getBasicBlock() ==
5226 &MF->getFunction().getEntryBlock()) {
5227 Register VReg = MI.getOperand(0).getReg();
5228 if (MF->getRegInfo().isLiveIn(VReg))
5229 return SignExt ? FuncInfo->isLiveInSExt(VReg) :
5230 FuncInfo->isLiveInZExt(VReg);
5231 }
5232
5233 // For a method return value, we check the ZExt/SExt flags in attribute.
5234 // We assume the following code sequence for method call.
5235 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
5236 // BL8_NOP @func,...
5237 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
5238 // %5 = COPY %x3; G8RC:%5
5239 if (SrcReg == PPC::X3) {
5240 const MachineBasicBlock *MBB = MI.getParent();
5241 MachineBasicBlock::const_instr_iterator II =
5242 MachineBasicBlock::const_instr_iterator(&MI);
5243 if (II != MBB->instr_begin() &&
5244 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) {
5245 const MachineInstr &CallMI = *(--II);
5246 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) {
5247 const Function *CalleeFn =
5248 dyn_cast<Function>(CallMI.getOperand(0).getGlobal());
5249 if (!CalleeFn)
5250 return false;
5251 const IntegerType *IntTy =
5252 dyn_cast<IntegerType>(CalleeFn->getReturnType());
5253 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
5254 if (IntTy && IntTy->getBitWidth() <= 32)
5255 return Attrs.hasAttribute(SignExt ? Attribute::SExt :
5256 Attribute::ZExt);
5257 }
5258 }
5259 }
5260 }
5261
5262 // If this is a copy from another register, we recursively check source.
5263 if (!Register::isVirtualRegister(SrcReg))
5264 return false;
5265 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
5266 if (SrcMI != NULL)
5267 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
5268
5269 return false;
5270 }
5271
5272 case PPC::ANDI_rec:
5273 case PPC::ANDIS_rec:
5274 case PPC::ORI:
5275 case PPC::ORIS:
5276 case PPC::XORI:
5277 case PPC::XORIS:
5278 case PPC::ANDI8_rec:
5279 case PPC::ANDIS8_rec:
5280 case PPC::ORI8:
5281 case PPC::ORIS8:
5282 case PPC::XORI8:
5283 case PPC::XORIS8: {
5284 // logical operation with 16-bit immediate does not change the upper bits.
5285 // So, we track the operand register as we do for register copy.
5286 Register SrcReg = MI.getOperand(1).getReg();
5287 if (!Register::isVirtualRegister(SrcReg))
5288 return false;
5289 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
5290 if (SrcMI != NULL)
5291 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
5292
5293 return false;
5294 }
5295
5296 // If all incoming values are sign-/zero-extended,
5297 // the output of OR, ISEL or PHI is also sign-/zero-extended.
5298 case PPC::OR:
5299 case PPC::OR8:
5300 case PPC::ISEL:
5301 case PPC::PHI: {
5302 if (Depth >= MAX_DEPTH)
5303 return false;
5304
5305 // The input registers for PHI are operand 1, 3, ...
5306 // The input registers for others are operand 1 and 2.
5307 unsigned E = 3, D = 1;
5308 if (MI.getOpcode() == PPC::PHI) {
5309 E = MI.getNumOperands();
5310 D = 2;
5311 }
5312
5313 for (unsigned I = 1; I != E; I += D) {
5314 if (MI.getOperand(I).isReg()) {
5315 Register SrcReg = MI.getOperand(I).getReg();
5316 if (!Register::isVirtualRegister(SrcReg))
5317 return false;
5318 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
5319 if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1))
5320 return false;
5321 }
5322 else
5323 return false;
5324 }
5325 return true;
5326 }
5327
5328 // If at least one of the incoming values of an AND is zero extended
5329 // then the output is also zero-extended. If both of the incoming values
5330 // are sign-extended then the output is also sign extended.
5331 case PPC::AND:
5332 case PPC::AND8: {
5333 if (Depth >= MAX_DEPTH)
5334 return false;
5335
5336 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg());
5337
5338 Register SrcReg1 = MI.getOperand(1).getReg();
5339 Register SrcReg2 = MI.getOperand(2).getReg();
5340
5341 if (!Register::isVirtualRegister(SrcReg1) ||
5342 !Register::isVirtualRegister(SrcReg2))
5343 return false;
5344
5345 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1);
5346 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2);
5347 if (!MISrc1 || !MISrc2)
5348 return false;
5349
5350 if(SignExt)
5351 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) &&
5352 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
5353 else
5354 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) ||
5355 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
5356 }
5357
5358 default:
5359 break;
5360 }
5361 return false;
5362 }
5363
isBDNZ(unsigned Opcode) const5364 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
5365 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
5366 }
5367
5368 namespace {
5369 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5370 MachineInstr *Loop, *EndLoop, *LoopCount;
5371 MachineFunction *MF;
5372 const TargetInstrInfo *TII;
5373 int64_t TripCount;
5374
5375 public:
PPCPipelinerLoopInfo(MachineInstr * Loop,MachineInstr * EndLoop,MachineInstr * LoopCount)5376 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
5377 MachineInstr *LoopCount)
5378 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
5379 MF(Loop->getParent()->getParent()),
5380 TII(MF->getSubtarget().getInstrInfo()) {
5381 // Inspect the Loop instruction up-front, as it may be deleted when we call
5382 // createTripCountGreaterCondition.
5383 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
5384 TripCount = LoopCount->getOperand(1).getImm();
5385 else
5386 TripCount = -1;
5387 }
5388
shouldIgnoreForPipelining(const MachineInstr * MI) const5389 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5390 // Only ignore the terminator.
5391 return MI == EndLoop;
5392 }
5393
5394 Optional<bool>
createTripCountGreaterCondition(int TC,MachineBasicBlock & MBB,SmallVectorImpl<MachineOperand> & Cond)5395 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
5396 SmallVectorImpl<MachineOperand> &Cond) override {
5397 if (TripCount == -1) {
5398 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5399 // so we don't need to generate any thing here.
5400 Cond.push_back(MachineOperand::CreateImm(0));
5401 Cond.push_back(MachineOperand::CreateReg(
5402 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
5403 true));
5404 return {};
5405 }
5406
5407 return TripCount > TC;
5408 }
5409
setPreheader(MachineBasicBlock * NewPreheader)5410 void setPreheader(MachineBasicBlock *NewPreheader) override {
5411 // Do nothing. We want the LOOP setup instruction to stay in the *old*
5412 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
5413 }
5414
adjustTripCount(int TripCountAdjust)5415 void adjustTripCount(int TripCountAdjust) override {
5416 // If the loop trip count is a compile-time value, then just change the
5417 // value.
5418 if (LoopCount->getOpcode() == PPC::LI8 ||
5419 LoopCount->getOpcode() == PPC::LI) {
5420 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
5421 LoopCount->getOperand(1).setImm(TripCount);
5422 return;
5423 }
5424
5425 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5426 // so we don't need to generate any thing here.
5427 }
5428
disposed()5429 void disposed() override {
5430 Loop->eraseFromParent();
5431 // Ensure the loop setup instruction is deleted too.
5432 LoopCount->eraseFromParent();
5433 }
5434 };
5435 } // namespace
5436
5437 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
analyzeLoopForPipelining(MachineBasicBlock * LoopBB) const5438 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5439 // We really "analyze" only hardware loops right now.
5440 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
5441 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
5442 if (Preheader == LoopBB)
5443 Preheader = *std::next(LoopBB->pred_begin());
5444 MachineFunction *MF = Preheader->getParent();
5445
5446 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
5447 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5448 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
5449 Register LoopCountReg = LoopInst->getOperand(0).getReg();
5450 MachineRegisterInfo &MRI = MF->getRegInfo();
5451 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
5452 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
5453 }
5454 }
5455 return nullptr;
5456 }
5457
findLoopInstr(MachineBasicBlock & PreHeader,SmallPtrSet<MachineBasicBlock *,8> & Visited) const5458 MachineInstr *PPCInstrInfo::findLoopInstr(
5459 MachineBasicBlock &PreHeader,
5460 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
5461
5462 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
5463
5464 // The loop set-up instruction should be in preheader
5465 for (auto &I : PreHeader.instrs())
5466 if (I.getOpcode() == LOOPi)
5467 return &I;
5468 return nullptr;
5469 }
5470
5471 // Return true if get the base operand, byte offset of an instruction and the
5472 // memory width. Width is the size of memory that is being loaded/stored.
getMemOperandWithOffsetWidth(const MachineInstr & LdSt,const MachineOperand * & BaseReg,int64_t & Offset,unsigned & Width,const TargetRegisterInfo * TRI) const5473 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
5474 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
5475 unsigned &Width, const TargetRegisterInfo *TRI) const {
5476 if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
5477 return false;
5478
5479 // Handle only loads/stores with base register followed by immediate offset.
5480 if (!LdSt.getOperand(1).isImm() ||
5481 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5482 return false;
5483 if (!LdSt.getOperand(1).isImm() ||
5484 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5485 return false;
5486
5487 if (!LdSt.hasOneMemOperand())
5488 return false;
5489
5490 Width = (*LdSt.memoperands_begin())->getSize();
5491 Offset = LdSt.getOperand(1).getImm();
5492 BaseReg = &LdSt.getOperand(2);
5493 return true;
5494 }
5495
areMemAccessesTriviallyDisjoint(const MachineInstr & MIa,const MachineInstr & MIb) const5496 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
5497 const MachineInstr &MIa, const MachineInstr &MIb) const {
5498 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
5499 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
5500
5501 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
5502 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
5503 return false;
5504
5505 // Retrieve the base register, offset from the base register and width. Width
5506 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
5507 // base registers are identical, and the offset of a lower memory access +
5508 // the width doesn't overlap the offset of a higher memory access,
5509 // then the memory accesses are different.
5510 const TargetRegisterInfo *TRI = &getRegisterInfo();
5511 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
5512 int64_t OffsetA = 0, OffsetB = 0;
5513 unsigned int WidthA = 0, WidthB = 0;
5514 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
5515 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
5516 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
5517 int LowOffset = std::min(OffsetA, OffsetB);
5518 int HighOffset = std::max(OffsetA, OffsetB);
5519 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
5520 if (LowOffset + LowWidth <= HighOffset)
5521 return true;
5522 }
5523 }
5524 return false;
5525 }
5526