1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/CodeGen/LiveIntervals.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineFunctionPass.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/PseudoSourceValue.h"
29 #include "llvm/CodeGen/ScheduleDAG.h"
30 #include "llvm/CodeGen/SlotIndexes.h"
31 #include "llvm/CodeGen/StackMaps.h"
32 #include "llvm/MC/MCAsmInfo.h"
33 #include "llvm/MC/MCInst.h"
34 #include "llvm/Support/CommandLine.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/ErrorHandling.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/raw_ostream.h"
39
40 using namespace llvm;
41
42 #define DEBUG_TYPE "ppc-instr-info"
43
44 #define GET_INSTRMAP_INFO
45 #define GET_INSTRINFO_CTOR_DTOR
46 #include "PPCGenInstrInfo.inc"
47
48 STATISTIC(NumStoreSPILLVSRRCAsVec,
49 "Number of spillvsrrc spilled to stack as vec");
50 STATISTIC(NumStoreSPILLVSRRCAsGpr,
51 "Number of spillvsrrc spilled to stack as gpr");
52 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
53 STATISTIC(CmpIselsConverted,
54 "Number of ISELs that depend on comparison of constants converted");
55 STATISTIC(MissedConvertibleImmediateInstrs,
56 "Number of compare-immediate instructions fed by constants");
57 STATISTIC(NumRcRotatesConvertedToRcAnd,
58 "Number of record-form rotates converted to record-form andi");
59
60 static cl::
61 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
62 cl::desc("Disable analysis for CTR loops"));
63
64 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
65 cl::desc("Disable compare instruction optimization"), cl::Hidden);
66
67 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
68 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
69 cl::Hidden);
70
71 static cl::opt<bool>
72 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
73 cl::desc("Use the old (incorrect) instruction latency calculation"));
74
75 // Index into the OpcodesForSpill array.
76 enum SpillOpcodeKey {
77 SOK_Int4Spill,
78 SOK_Int8Spill,
79 SOK_Float8Spill,
80 SOK_Float4Spill,
81 SOK_CRSpill,
82 SOK_CRBitSpill,
83 SOK_VRVectorSpill,
84 SOK_VSXVectorSpill,
85 SOK_VectorFloat8Spill,
86 SOK_VectorFloat4Spill,
87 SOK_VRSaveSpill,
88 SOK_QuadFloat8Spill,
89 SOK_QuadFloat4Spill,
90 SOK_QuadBitSpill,
91 SOK_SpillToVSR,
92 SOK_SPESpill,
93 SOK_LastOpcodeSpill // This must be last on the enum.
94 };
95
96 // Pin the vtable to this file.
anchor()97 void PPCInstrInfo::anchor() {}
98
PPCInstrInfo(PPCSubtarget & STI)99 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
100 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
101 /* CatchRetOpcode */ -1,
102 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
103 Subtarget(STI), RI(STI.getTargetMachine()) {}
104
105 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
106 /// this target when scheduling the DAG.
107 ScheduleHazardRecognizer *
CreateTargetHazardRecognizer(const TargetSubtargetInfo * STI,const ScheduleDAG * DAG) const108 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
109 const ScheduleDAG *DAG) const {
110 unsigned Directive =
111 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
112 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
113 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
114 const InstrItineraryData *II =
115 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
116 return new ScoreboardHazardRecognizer(II, DAG);
117 }
118
119 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
120 }
121
122 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
123 /// to use for this target when scheduling the DAG.
124 ScheduleHazardRecognizer *
CreateTargetPostRAHazardRecognizer(const InstrItineraryData * II,const ScheduleDAG * DAG) const125 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
126 const ScheduleDAG *DAG) const {
127 unsigned Directive =
128 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
129
130 // FIXME: Leaving this as-is until we have POWER9 scheduling info
131 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
132 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
133
134 // Most subtargets use a PPC970 recognizer.
135 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
136 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
137 assert(DAG->TII && "No InstrInfo?");
138
139 return new PPCHazardRecognizer970(*DAG);
140 }
141
142 return new ScoreboardHazardRecognizer(II, DAG);
143 }
144
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr & MI,unsigned * PredCost) const145 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
146 const MachineInstr &MI,
147 unsigned *PredCost) const {
148 if (!ItinData || UseOldLatencyCalc)
149 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
150
151 // The default implementation of getInstrLatency calls getStageLatency, but
152 // getStageLatency does not do the right thing for us. While we have
153 // itinerary, most cores are fully pipelined, and so the itineraries only
154 // express the first part of the pipeline, not every stage. Instead, we need
155 // to use the listed output operand cycle number (using operand 0 here, which
156 // is an output).
157
158 unsigned Latency = 1;
159 unsigned DefClass = MI.getDesc().getSchedClass();
160 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
161 const MachineOperand &MO = MI.getOperand(i);
162 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
163 continue;
164
165 int Cycle = ItinData->getOperandCycle(DefClass, i);
166 if (Cycle < 0)
167 continue;
168
169 Latency = std::max(Latency, (unsigned) Cycle);
170 }
171
172 return Latency;
173 }
174
getOperandLatency(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const175 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
176 const MachineInstr &DefMI, unsigned DefIdx,
177 const MachineInstr &UseMI,
178 unsigned UseIdx) const {
179 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
180 UseMI, UseIdx);
181
182 if (!DefMI.getParent())
183 return Latency;
184
185 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
186 Register Reg = DefMO.getReg();
187
188 bool IsRegCR;
189 if (Register::isVirtualRegister(Reg)) {
190 const MachineRegisterInfo *MRI =
191 &DefMI.getParent()->getParent()->getRegInfo();
192 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
193 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
194 } else {
195 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
196 PPC::CRBITRCRegClass.contains(Reg);
197 }
198
199 if (UseMI.isBranch() && IsRegCR) {
200 if (Latency < 0)
201 Latency = getInstrLatency(ItinData, DefMI);
202
203 // On some cores, there is an additional delay between writing to a condition
204 // register, and using it from a branch.
205 unsigned Directive = Subtarget.getCPUDirective();
206 switch (Directive) {
207 default: break;
208 case PPC::DIR_7400:
209 case PPC::DIR_750:
210 case PPC::DIR_970:
211 case PPC::DIR_E5500:
212 case PPC::DIR_PWR4:
213 case PPC::DIR_PWR5:
214 case PPC::DIR_PWR5X:
215 case PPC::DIR_PWR6:
216 case PPC::DIR_PWR6X:
217 case PPC::DIR_PWR7:
218 case PPC::DIR_PWR8:
219 // FIXME: Is this needed for POWER9?
220 Latency += 2;
221 break;
222 }
223 }
224
225 return Latency;
226 }
227
228 // This function does not list all associative and commutative operations, but
229 // only those worth feeding through the machine combiner in an attempt to
230 // reduce the critical path. Mostly, this means floating-point operations,
231 // because they have high latencies (compared to other operations, such and
232 // and/or, which are also associative and commutative, but have low latencies).
isAssociativeAndCommutative(const MachineInstr & Inst) const233 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
234 switch (Inst.getOpcode()) {
235 // FP Add:
236 case PPC::FADD:
237 case PPC::FADDS:
238 // FP Multiply:
239 case PPC::FMUL:
240 case PPC::FMULS:
241 // Altivec Add:
242 case PPC::VADDFP:
243 // VSX Add:
244 case PPC::XSADDDP:
245 case PPC::XVADDDP:
246 case PPC::XVADDSP:
247 case PPC::XSADDSP:
248 // VSX Multiply:
249 case PPC::XSMULDP:
250 case PPC::XVMULDP:
251 case PPC::XVMULSP:
252 case PPC::XSMULSP:
253 // QPX Add:
254 case PPC::QVFADD:
255 case PPC::QVFADDS:
256 case PPC::QVFADDSs:
257 // QPX Multiply:
258 case PPC::QVFMUL:
259 case PPC::QVFMULS:
260 case PPC::QVFMULSs:
261 return true;
262 default:
263 return false;
264 }
265 }
266
getMachineCombinerPatterns(MachineInstr & Root,SmallVectorImpl<MachineCombinerPattern> & Patterns) const267 bool PPCInstrInfo::getMachineCombinerPatterns(
268 MachineInstr &Root,
269 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
270 // Using the machine combiner in this way is potentially expensive, so
271 // restrict to when aggressive optimizations are desired.
272 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
273 return false;
274
275 // FP reassociation is only legal when we don't need strict IEEE semantics.
276 if (!Root.getParent()->getParent()->getTarget().Options.UnsafeFPMath)
277 return false;
278
279 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
280 }
281
282 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
isCoalescableExtInstr(const MachineInstr & MI,unsigned & SrcReg,unsigned & DstReg,unsigned & SubIdx) const283 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
284 unsigned &SrcReg, unsigned &DstReg,
285 unsigned &SubIdx) const {
286 switch (MI.getOpcode()) {
287 default: return false;
288 case PPC::EXTSW:
289 case PPC::EXTSW_32:
290 case PPC::EXTSW_32_64:
291 SrcReg = MI.getOperand(1).getReg();
292 DstReg = MI.getOperand(0).getReg();
293 SubIdx = PPC::sub_32;
294 return true;
295 }
296 }
297
isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const298 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
299 int &FrameIndex) const {
300 unsigned Opcode = MI.getOpcode();
301 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
302 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
303
304 if (End != std::find(OpcodesForSpill, End, Opcode)) {
305 // Check for the operands added by addFrameReference (the immediate is the
306 // offset which defaults to 0).
307 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
308 MI.getOperand(2).isFI()) {
309 FrameIndex = MI.getOperand(2).getIndex();
310 return MI.getOperand(0).getReg();
311 }
312 }
313 return 0;
314 }
315
316 // For opcodes with the ReMaterializable flag set, this function is called to
317 // verify the instruction is really rematable.
isReallyTriviallyReMaterializable(const MachineInstr & MI,AliasAnalysis * AA) const318 bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
319 AliasAnalysis *AA) const {
320 switch (MI.getOpcode()) {
321 default:
322 // This function should only be called for opcodes with the ReMaterializable
323 // flag set.
324 llvm_unreachable("Unknown rematerializable operation!");
325 break;
326 case PPC::LI:
327 case PPC::LI8:
328 case PPC::LIS:
329 case PPC::LIS8:
330 case PPC::QVGPCI:
331 case PPC::ADDIStocHA:
332 case PPC::ADDIStocHA8:
333 case PPC::ADDItocL:
334 case PPC::LOAD_STACK_GUARD:
335 case PPC::XXLXORz:
336 case PPC::XXLXORspz:
337 case PPC::XXLXORdpz:
338 case PPC::XXLEQVOnes:
339 case PPC::V_SET0B:
340 case PPC::V_SET0H:
341 case PPC::V_SET0:
342 case PPC::V_SETALLONESB:
343 case PPC::V_SETALLONESH:
344 case PPC::V_SETALLONES:
345 case PPC::CRSET:
346 case PPC::CRUNSET:
347 return true;
348 }
349 return false;
350 }
351
isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const352 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
353 int &FrameIndex) const {
354 unsigned Opcode = MI.getOpcode();
355 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
356 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
357
358 if (End != std::find(OpcodesForSpill, End, Opcode)) {
359 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
360 MI.getOperand(2).isFI()) {
361 FrameIndex = MI.getOperand(2).getIndex();
362 return MI.getOperand(0).getReg();
363 }
364 }
365 return 0;
366 }
367
commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const368 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
369 unsigned OpIdx1,
370 unsigned OpIdx2) const {
371 MachineFunction &MF = *MI.getParent()->getParent();
372
373 // Normal instructions can be commuted the obvious way.
374 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
375 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
376 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
377 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
378 // changing the relative order of the mask operands might change what happens
379 // to the high-bits of the mask (and, thus, the result).
380
381 // Cannot commute if it has a non-zero rotate count.
382 if (MI.getOperand(3).getImm() != 0)
383 return nullptr;
384
385 // If we have a zero rotate count, we have:
386 // M = mask(MB,ME)
387 // Op0 = (Op1 & ~M) | (Op2 & M)
388 // Change this to:
389 // M = mask((ME+1)&31, (MB-1)&31)
390 // Op0 = (Op2 & ~M) | (Op1 & M)
391
392 // Swap op1/op2
393 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
394 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
395 Register Reg0 = MI.getOperand(0).getReg();
396 Register Reg1 = MI.getOperand(1).getReg();
397 Register Reg2 = MI.getOperand(2).getReg();
398 unsigned SubReg1 = MI.getOperand(1).getSubReg();
399 unsigned SubReg2 = MI.getOperand(2).getSubReg();
400 bool Reg1IsKill = MI.getOperand(1).isKill();
401 bool Reg2IsKill = MI.getOperand(2).isKill();
402 bool ChangeReg0 = false;
403 // If machine instrs are no longer in two-address forms, update
404 // destination register as well.
405 if (Reg0 == Reg1) {
406 // Must be two address instruction!
407 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
408 "Expecting a two-address instruction!");
409 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
410 Reg2IsKill = false;
411 ChangeReg0 = true;
412 }
413
414 // Masks.
415 unsigned MB = MI.getOperand(4).getImm();
416 unsigned ME = MI.getOperand(5).getImm();
417
418 // We can't commute a trivial mask (there is no way to represent an all-zero
419 // mask).
420 if (MB == 0 && ME == 31)
421 return nullptr;
422
423 if (NewMI) {
424 // Create a new instruction.
425 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
426 bool Reg0IsDead = MI.getOperand(0).isDead();
427 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
428 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
429 .addReg(Reg2, getKillRegState(Reg2IsKill))
430 .addReg(Reg1, getKillRegState(Reg1IsKill))
431 .addImm((ME + 1) & 31)
432 .addImm((MB - 1) & 31);
433 }
434
435 if (ChangeReg0) {
436 MI.getOperand(0).setReg(Reg2);
437 MI.getOperand(0).setSubReg(SubReg2);
438 }
439 MI.getOperand(2).setReg(Reg1);
440 MI.getOperand(1).setReg(Reg2);
441 MI.getOperand(2).setSubReg(SubReg1);
442 MI.getOperand(1).setSubReg(SubReg2);
443 MI.getOperand(2).setIsKill(Reg1IsKill);
444 MI.getOperand(1).setIsKill(Reg2IsKill);
445
446 // Swap the mask around.
447 MI.getOperand(4).setImm((ME + 1) & 31);
448 MI.getOperand(5).setImm((MB - 1) & 31);
449 return &MI;
450 }
451
findCommutedOpIndices(const MachineInstr & MI,unsigned & SrcOpIdx1,unsigned & SrcOpIdx2) const452 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
453 unsigned &SrcOpIdx1,
454 unsigned &SrcOpIdx2) const {
455 // For VSX A-Type FMA instructions, it is the first two operands that can be
456 // commuted, however, because the non-encoded tied input operand is listed
457 // first, the operands to swap are actually the second and third.
458
459 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
460 if (AltOpc == -1)
461 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
462
463 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
464 // and SrcOpIdx2.
465 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
466 }
467
insertNoop(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const468 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
469 MachineBasicBlock::iterator MI) const {
470 // This function is used for scheduling, and the nop wanted here is the type
471 // that terminates dispatch groups on the POWER cores.
472 unsigned Directive = Subtarget.getCPUDirective();
473 unsigned Opcode;
474 switch (Directive) {
475 default: Opcode = PPC::NOP; break;
476 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
477 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
478 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
479 // FIXME: Update when POWER9 scheduling model is ready.
480 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
481 }
482
483 DebugLoc DL;
484 BuildMI(MBB, MI, DL, get(Opcode));
485 }
486
487 /// Return the noop instruction to use for a noop.
getNoop(MCInst & NopInst) const488 void PPCInstrInfo::getNoop(MCInst &NopInst) const {
489 NopInst.setOpcode(PPC::NOP);
490 }
491
492 // Branch analysis.
493 // Note: If the condition register is set to CTR or CTR8 then this is a
494 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const495 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
496 MachineBasicBlock *&TBB,
497 MachineBasicBlock *&FBB,
498 SmallVectorImpl<MachineOperand> &Cond,
499 bool AllowModify) const {
500 bool isPPC64 = Subtarget.isPPC64();
501
502 // If the block has no terminators, it just falls into the block after it.
503 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
504 if (I == MBB.end())
505 return false;
506
507 if (!isUnpredicatedTerminator(*I))
508 return false;
509
510 if (AllowModify) {
511 // If the BB ends with an unconditional branch to the fallthrough BB,
512 // we eliminate the branch instruction.
513 if (I->getOpcode() == PPC::B &&
514 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
515 I->eraseFromParent();
516
517 // We update iterator after deleting the last branch.
518 I = MBB.getLastNonDebugInstr();
519 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
520 return false;
521 }
522 }
523
524 // Get the last instruction in the block.
525 MachineInstr &LastInst = *I;
526
527 // If there is only one terminator instruction, process it.
528 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
529 if (LastInst.getOpcode() == PPC::B) {
530 if (!LastInst.getOperand(0).isMBB())
531 return true;
532 TBB = LastInst.getOperand(0).getMBB();
533 return false;
534 } else if (LastInst.getOpcode() == PPC::BCC) {
535 if (!LastInst.getOperand(2).isMBB())
536 return true;
537 // Block ends with fall-through condbranch.
538 TBB = LastInst.getOperand(2).getMBB();
539 Cond.push_back(LastInst.getOperand(0));
540 Cond.push_back(LastInst.getOperand(1));
541 return false;
542 } else if (LastInst.getOpcode() == PPC::BC) {
543 if (!LastInst.getOperand(1).isMBB())
544 return true;
545 // Block ends with fall-through condbranch.
546 TBB = LastInst.getOperand(1).getMBB();
547 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
548 Cond.push_back(LastInst.getOperand(0));
549 return false;
550 } else if (LastInst.getOpcode() == PPC::BCn) {
551 if (!LastInst.getOperand(1).isMBB())
552 return true;
553 // Block ends with fall-through condbranch.
554 TBB = LastInst.getOperand(1).getMBB();
555 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
556 Cond.push_back(LastInst.getOperand(0));
557 return false;
558 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
559 LastInst.getOpcode() == PPC::BDNZ) {
560 if (!LastInst.getOperand(0).isMBB())
561 return true;
562 if (DisableCTRLoopAnal)
563 return true;
564 TBB = LastInst.getOperand(0).getMBB();
565 Cond.push_back(MachineOperand::CreateImm(1));
566 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
567 true));
568 return false;
569 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
570 LastInst.getOpcode() == PPC::BDZ) {
571 if (!LastInst.getOperand(0).isMBB())
572 return true;
573 if (DisableCTRLoopAnal)
574 return true;
575 TBB = LastInst.getOperand(0).getMBB();
576 Cond.push_back(MachineOperand::CreateImm(0));
577 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
578 true));
579 return false;
580 }
581
582 // Otherwise, don't know what this is.
583 return true;
584 }
585
586 // Get the instruction before it if it's a terminator.
587 MachineInstr &SecondLastInst = *I;
588
589 // If there are three terminators, we don't know what sort of block this is.
590 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
591 return true;
592
593 // If the block ends with PPC::B and PPC:BCC, handle it.
594 if (SecondLastInst.getOpcode() == PPC::BCC &&
595 LastInst.getOpcode() == PPC::B) {
596 if (!SecondLastInst.getOperand(2).isMBB() ||
597 !LastInst.getOperand(0).isMBB())
598 return true;
599 TBB = SecondLastInst.getOperand(2).getMBB();
600 Cond.push_back(SecondLastInst.getOperand(0));
601 Cond.push_back(SecondLastInst.getOperand(1));
602 FBB = LastInst.getOperand(0).getMBB();
603 return false;
604 } else if (SecondLastInst.getOpcode() == PPC::BC &&
605 LastInst.getOpcode() == PPC::B) {
606 if (!SecondLastInst.getOperand(1).isMBB() ||
607 !LastInst.getOperand(0).isMBB())
608 return true;
609 TBB = SecondLastInst.getOperand(1).getMBB();
610 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
611 Cond.push_back(SecondLastInst.getOperand(0));
612 FBB = LastInst.getOperand(0).getMBB();
613 return false;
614 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
615 LastInst.getOpcode() == PPC::B) {
616 if (!SecondLastInst.getOperand(1).isMBB() ||
617 !LastInst.getOperand(0).isMBB())
618 return true;
619 TBB = SecondLastInst.getOperand(1).getMBB();
620 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
621 Cond.push_back(SecondLastInst.getOperand(0));
622 FBB = LastInst.getOperand(0).getMBB();
623 return false;
624 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
625 SecondLastInst.getOpcode() == PPC::BDNZ) &&
626 LastInst.getOpcode() == PPC::B) {
627 if (!SecondLastInst.getOperand(0).isMBB() ||
628 !LastInst.getOperand(0).isMBB())
629 return true;
630 if (DisableCTRLoopAnal)
631 return true;
632 TBB = SecondLastInst.getOperand(0).getMBB();
633 Cond.push_back(MachineOperand::CreateImm(1));
634 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
635 true));
636 FBB = LastInst.getOperand(0).getMBB();
637 return false;
638 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
639 SecondLastInst.getOpcode() == PPC::BDZ) &&
640 LastInst.getOpcode() == PPC::B) {
641 if (!SecondLastInst.getOperand(0).isMBB() ||
642 !LastInst.getOperand(0).isMBB())
643 return true;
644 if (DisableCTRLoopAnal)
645 return true;
646 TBB = SecondLastInst.getOperand(0).getMBB();
647 Cond.push_back(MachineOperand::CreateImm(0));
648 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
649 true));
650 FBB = LastInst.getOperand(0).getMBB();
651 return false;
652 }
653
654 // If the block ends with two PPC:Bs, handle it. The second one is not
655 // executed, so remove it.
656 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
657 if (!SecondLastInst.getOperand(0).isMBB())
658 return true;
659 TBB = SecondLastInst.getOperand(0).getMBB();
660 I = LastInst;
661 if (AllowModify)
662 I->eraseFromParent();
663 return false;
664 }
665
666 // Otherwise, can't handle this.
667 return true;
668 }
669
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const670 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
671 int *BytesRemoved) const {
672 assert(!BytesRemoved && "code size not handled");
673
674 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
675 if (I == MBB.end())
676 return 0;
677
678 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
679 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
680 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
681 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
682 return 0;
683
684 // Remove the branch.
685 I->eraseFromParent();
686
687 I = MBB.end();
688
689 if (I == MBB.begin()) return 1;
690 --I;
691 if (I->getOpcode() != PPC::BCC &&
692 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
693 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
694 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
695 return 1;
696
697 // Remove the branch.
698 I->eraseFromParent();
699 return 2;
700 }
701
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const702 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
703 MachineBasicBlock *TBB,
704 MachineBasicBlock *FBB,
705 ArrayRef<MachineOperand> Cond,
706 const DebugLoc &DL,
707 int *BytesAdded) const {
708 // Shouldn't be a fall through.
709 assert(TBB && "insertBranch must not be told to insert a fallthrough");
710 assert((Cond.size() == 2 || Cond.size() == 0) &&
711 "PPC branch conditions have two components!");
712 assert(!BytesAdded && "code size not handled");
713
714 bool isPPC64 = Subtarget.isPPC64();
715
716 // One-way branch.
717 if (!FBB) {
718 if (Cond.empty()) // Unconditional branch
719 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
720 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
721 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
722 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
723 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
724 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
725 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
726 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
727 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
728 else // Conditional branch
729 BuildMI(&MBB, DL, get(PPC::BCC))
730 .addImm(Cond[0].getImm())
731 .add(Cond[1])
732 .addMBB(TBB);
733 return 1;
734 }
735
736 // Two-way Conditional Branch.
737 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
738 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
739 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
740 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
741 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
742 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
743 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
744 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
745 else
746 BuildMI(&MBB, DL, get(PPC::BCC))
747 .addImm(Cond[0].getImm())
748 .add(Cond[1])
749 .addMBB(TBB);
750 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
751 return 2;
752 }
753
754 // Select analysis.
canInsertSelect(const MachineBasicBlock & MBB,ArrayRef<MachineOperand> Cond,unsigned TrueReg,unsigned FalseReg,int & CondCycles,int & TrueCycles,int & FalseCycles) const755 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
756 ArrayRef<MachineOperand> Cond,
757 unsigned TrueReg, unsigned FalseReg,
758 int &CondCycles, int &TrueCycles, int &FalseCycles) const {
759 if (Cond.size() != 2)
760 return false;
761
762 // If this is really a bdnz-like condition, then it cannot be turned into a
763 // select.
764 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
765 return false;
766
767 // Check register classes.
768 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
769 const TargetRegisterClass *RC =
770 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
771 if (!RC)
772 return false;
773
774 // isel is for regular integer GPRs only.
775 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
776 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
777 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
778 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
779 return false;
780
781 // FIXME: These numbers are for the A2, how well they work for other cores is
782 // an open question. On the A2, the isel instruction has a 2-cycle latency
783 // but single-cycle throughput. These numbers are used in combination with
784 // the MispredictPenalty setting from the active SchedMachineModel.
785 CondCycles = 1;
786 TrueCycles = 1;
787 FalseCycles = 1;
788
789 return true;
790 }
791
insertSelect(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & dl,unsigned DestReg,ArrayRef<MachineOperand> Cond,unsigned TrueReg,unsigned FalseReg) const792 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
793 MachineBasicBlock::iterator MI,
794 const DebugLoc &dl, unsigned DestReg,
795 ArrayRef<MachineOperand> Cond, unsigned TrueReg,
796 unsigned FalseReg) const {
797 assert(Cond.size() == 2 &&
798 "PPC branch conditions have two components!");
799
800 // Get the register classes.
801 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
802 const TargetRegisterClass *RC =
803 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
804 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
805
806 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
807 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
808 assert((Is64Bit ||
809 PPC::GPRCRegClass.hasSubClassEq(RC) ||
810 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
811 "isel is for regular integer GPRs only");
812
813 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
814 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
815
816 unsigned SubIdx = 0;
817 bool SwapOps = false;
818 switch (SelectPred) {
819 case PPC::PRED_EQ:
820 case PPC::PRED_EQ_MINUS:
821 case PPC::PRED_EQ_PLUS:
822 SubIdx = PPC::sub_eq; SwapOps = false; break;
823 case PPC::PRED_NE:
824 case PPC::PRED_NE_MINUS:
825 case PPC::PRED_NE_PLUS:
826 SubIdx = PPC::sub_eq; SwapOps = true; break;
827 case PPC::PRED_LT:
828 case PPC::PRED_LT_MINUS:
829 case PPC::PRED_LT_PLUS:
830 SubIdx = PPC::sub_lt; SwapOps = false; break;
831 case PPC::PRED_GE:
832 case PPC::PRED_GE_MINUS:
833 case PPC::PRED_GE_PLUS:
834 SubIdx = PPC::sub_lt; SwapOps = true; break;
835 case PPC::PRED_GT:
836 case PPC::PRED_GT_MINUS:
837 case PPC::PRED_GT_PLUS:
838 SubIdx = PPC::sub_gt; SwapOps = false; break;
839 case PPC::PRED_LE:
840 case PPC::PRED_LE_MINUS:
841 case PPC::PRED_LE_PLUS:
842 SubIdx = PPC::sub_gt; SwapOps = true; break;
843 case PPC::PRED_UN:
844 case PPC::PRED_UN_MINUS:
845 case PPC::PRED_UN_PLUS:
846 SubIdx = PPC::sub_un; SwapOps = false; break;
847 case PPC::PRED_NU:
848 case PPC::PRED_NU_MINUS:
849 case PPC::PRED_NU_PLUS:
850 SubIdx = PPC::sub_un; SwapOps = true; break;
851 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
852 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
853 }
854
855 unsigned FirstReg = SwapOps ? FalseReg : TrueReg,
856 SecondReg = SwapOps ? TrueReg : FalseReg;
857
858 // The first input register of isel cannot be r0. If it is a member
859 // of a register class that can be r0, then copy it first (the
860 // register allocator should eliminate the copy).
861 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
862 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
863 const TargetRegisterClass *FirstRC =
864 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
865 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
866 unsigned OldFirstReg = FirstReg;
867 FirstReg = MRI.createVirtualRegister(FirstRC);
868 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
869 .addReg(OldFirstReg);
870 }
871
872 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
873 .addReg(FirstReg).addReg(SecondReg)
874 .addReg(Cond[1].getReg(), 0, SubIdx);
875 }
876
getCRBitValue(unsigned CRBit)877 static unsigned getCRBitValue(unsigned CRBit) {
878 unsigned Ret = 4;
879 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
880 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
881 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
882 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
883 Ret = 3;
884 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
885 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
886 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
887 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
888 Ret = 2;
889 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
890 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
891 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
892 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
893 Ret = 1;
894 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
895 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
896 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
897 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
898 Ret = 0;
899
900 assert(Ret != 4 && "Invalid CR bit register");
901 return Ret;
902 }
903
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const904 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
905 MachineBasicBlock::iterator I,
906 const DebugLoc &DL, MCRegister DestReg,
907 MCRegister SrcReg, bool KillSrc) const {
908 // We can end up with self copies and similar things as a result of VSX copy
909 // legalization. Promote them here.
910 const TargetRegisterInfo *TRI = &getRegisterInfo();
911 if (PPC::F8RCRegClass.contains(DestReg) &&
912 PPC::VSRCRegClass.contains(SrcReg)) {
913 MCRegister SuperReg =
914 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
915
916 if (VSXSelfCopyCrash && SrcReg == SuperReg)
917 llvm_unreachable("nop VSX copy");
918
919 DestReg = SuperReg;
920 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
921 PPC::VSRCRegClass.contains(DestReg)) {
922 MCRegister SuperReg =
923 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
924
925 if (VSXSelfCopyCrash && DestReg == SuperReg)
926 llvm_unreachable("nop VSX copy");
927
928 SrcReg = SuperReg;
929 }
930
931 // Different class register copy
932 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
933 PPC::GPRCRegClass.contains(DestReg)) {
934 MCRegister CRReg = getCRFromCRBit(SrcReg);
935 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
936 getKillRegState(KillSrc);
937 // Rotate the CR bit in the CR fields to be the least significant bit and
938 // then mask with 0x1 (MB = ME = 31).
939 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
940 .addReg(DestReg, RegState::Kill)
941 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
942 .addImm(31)
943 .addImm(31);
944 return;
945 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
946 PPC::G8RCRegClass.contains(DestReg)) {
947 BuildMI(MBB, I, DL, get(PPC::MFOCRF8), DestReg).addReg(SrcReg);
948 getKillRegState(KillSrc);
949 return;
950 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
951 PPC::GPRCRegClass.contains(DestReg)) {
952 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(SrcReg);
953 getKillRegState(KillSrc);
954 return;
955 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
956 PPC::VSFRCRegClass.contains(DestReg)) {
957 assert(Subtarget.hasDirectMove() &&
958 "Subtarget doesn't support directmove, don't know how to copy.");
959 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
960 NumGPRtoVSRSpill++;
961 getKillRegState(KillSrc);
962 return;
963 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
964 PPC::G8RCRegClass.contains(DestReg)) {
965 assert(Subtarget.hasDirectMove() &&
966 "Subtarget doesn't support directmove, don't know how to copy.");
967 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
968 getKillRegState(KillSrc);
969 return;
970 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
971 PPC::GPRCRegClass.contains(DestReg)) {
972 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
973 getKillRegState(KillSrc);
974 return;
975 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
976 PPC::SPERCRegClass.contains(DestReg)) {
977 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
978 getKillRegState(KillSrc);
979 return;
980 }
981
982 unsigned Opc;
983 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
984 Opc = PPC::OR;
985 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
986 Opc = PPC::OR8;
987 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
988 Opc = PPC::FMR;
989 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
990 Opc = PPC::MCRF;
991 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
992 Opc = PPC::VOR;
993 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
994 // There are two different ways this can be done:
995 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
996 // issue in VSU pipeline 0.
997 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
998 // can go to either pipeline.
999 // We'll always use xxlor here, because in practically all cases where
1000 // copies are generated, they are close enough to some use that the
1001 // lower-latency form is preferable.
1002 Opc = PPC::XXLOR;
1003 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1004 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1005 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1006 else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
1007 Opc = PPC::QVFMR;
1008 else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
1009 Opc = PPC::QVFMRs;
1010 else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
1011 Opc = PPC::QVFMRb;
1012 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1013 Opc = PPC::CROR;
1014 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1015 Opc = PPC::EVOR;
1016 else
1017 llvm_unreachable("Impossible reg-to-reg copy");
1018
1019 const MCInstrDesc &MCID = get(Opc);
1020 if (MCID.getNumOperands() == 3)
1021 BuildMI(MBB, I, DL, MCID, DestReg)
1022 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1023 else
1024 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1025 }
1026
getStoreOpcodeForSpill(unsigned Reg,const TargetRegisterClass * RC) const1027 unsigned PPCInstrInfo::getStoreOpcodeForSpill(unsigned Reg,
1028 const TargetRegisterClass *RC)
1029 const {
1030 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1031 int OpcodeIndex = 0;
1032
1033 if (RC != nullptr) {
1034 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1035 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1036 OpcodeIndex = SOK_Int4Spill;
1037 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1038 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1039 OpcodeIndex = SOK_Int8Spill;
1040 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1041 OpcodeIndex = SOK_Float8Spill;
1042 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1043 OpcodeIndex = SOK_Float4Spill;
1044 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1045 OpcodeIndex = SOK_SPESpill;
1046 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1047 OpcodeIndex = SOK_CRSpill;
1048 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1049 OpcodeIndex = SOK_CRBitSpill;
1050 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1051 OpcodeIndex = SOK_VRVectorSpill;
1052 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1053 OpcodeIndex = SOK_VSXVectorSpill;
1054 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1055 OpcodeIndex = SOK_VectorFloat8Spill;
1056 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1057 OpcodeIndex = SOK_VectorFloat4Spill;
1058 } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
1059 OpcodeIndex = SOK_VRSaveSpill;
1060 } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
1061 OpcodeIndex = SOK_QuadFloat8Spill;
1062 } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
1063 OpcodeIndex = SOK_QuadFloat4Spill;
1064 } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
1065 OpcodeIndex = SOK_QuadBitSpill;
1066 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1067 OpcodeIndex = SOK_SpillToVSR;
1068 } else {
1069 llvm_unreachable("Unknown regclass!");
1070 }
1071 } else {
1072 if (PPC::GPRCRegClass.contains(Reg) ||
1073 PPC::GPRC_NOR0RegClass.contains(Reg)) {
1074 OpcodeIndex = SOK_Int4Spill;
1075 } else if (PPC::G8RCRegClass.contains(Reg) ||
1076 PPC::G8RC_NOX0RegClass.contains(Reg)) {
1077 OpcodeIndex = SOK_Int8Spill;
1078 } else if (PPC::F8RCRegClass.contains(Reg)) {
1079 OpcodeIndex = SOK_Float8Spill;
1080 } else if (PPC::F4RCRegClass.contains(Reg)) {
1081 OpcodeIndex = SOK_Float4Spill;
1082 } else if (PPC::SPERCRegClass.contains(Reg)) {
1083 OpcodeIndex = SOK_SPESpill;
1084 } else if (PPC::CRRCRegClass.contains(Reg)) {
1085 OpcodeIndex = SOK_CRSpill;
1086 } else if (PPC::CRBITRCRegClass.contains(Reg)) {
1087 OpcodeIndex = SOK_CRBitSpill;
1088 } else if (PPC::VRRCRegClass.contains(Reg)) {
1089 OpcodeIndex = SOK_VRVectorSpill;
1090 } else if (PPC::VSRCRegClass.contains(Reg)) {
1091 OpcodeIndex = SOK_VSXVectorSpill;
1092 } else if (PPC::VSFRCRegClass.contains(Reg)) {
1093 OpcodeIndex = SOK_VectorFloat8Spill;
1094 } else if (PPC::VSSRCRegClass.contains(Reg)) {
1095 OpcodeIndex = SOK_VectorFloat4Spill;
1096 } else if (PPC::VRSAVERCRegClass.contains(Reg)) {
1097 OpcodeIndex = SOK_VRSaveSpill;
1098 } else if (PPC::QFRCRegClass.contains(Reg)) {
1099 OpcodeIndex = SOK_QuadFloat8Spill;
1100 } else if (PPC::QSRCRegClass.contains(Reg)) {
1101 OpcodeIndex = SOK_QuadFloat4Spill;
1102 } else if (PPC::QBRCRegClass.contains(Reg)) {
1103 OpcodeIndex = SOK_QuadBitSpill;
1104 } else if (PPC::SPILLTOVSRRCRegClass.contains(Reg)) {
1105 OpcodeIndex = SOK_SpillToVSR;
1106 } else {
1107 llvm_unreachable("Unknown regclass!");
1108 }
1109 }
1110 return OpcodesForSpill[OpcodeIndex];
1111 }
1112
1113 unsigned
getLoadOpcodeForSpill(unsigned Reg,const TargetRegisterClass * RC) const1114 PPCInstrInfo::getLoadOpcodeForSpill(unsigned Reg,
1115 const TargetRegisterClass *RC) const {
1116 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1117 int OpcodeIndex = 0;
1118
1119 if (RC != nullptr) {
1120 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1121 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1122 OpcodeIndex = SOK_Int4Spill;
1123 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1124 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1125 OpcodeIndex = SOK_Int8Spill;
1126 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1127 OpcodeIndex = SOK_Float8Spill;
1128 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1129 OpcodeIndex = SOK_Float4Spill;
1130 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1131 OpcodeIndex = SOK_SPESpill;
1132 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1133 OpcodeIndex = SOK_CRSpill;
1134 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1135 OpcodeIndex = SOK_CRBitSpill;
1136 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1137 OpcodeIndex = SOK_VRVectorSpill;
1138 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1139 OpcodeIndex = SOK_VSXVectorSpill;
1140 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1141 OpcodeIndex = SOK_VectorFloat8Spill;
1142 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1143 OpcodeIndex = SOK_VectorFloat4Spill;
1144 } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
1145 OpcodeIndex = SOK_VRSaveSpill;
1146 } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
1147 OpcodeIndex = SOK_QuadFloat8Spill;
1148 } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
1149 OpcodeIndex = SOK_QuadFloat4Spill;
1150 } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
1151 OpcodeIndex = SOK_QuadBitSpill;
1152 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1153 OpcodeIndex = SOK_SpillToVSR;
1154 } else {
1155 llvm_unreachable("Unknown regclass!");
1156 }
1157 } else {
1158 if (PPC::GPRCRegClass.contains(Reg) ||
1159 PPC::GPRC_NOR0RegClass.contains(Reg)) {
1160 OpcodeIndex = SOK_Int4Spill;
1161 } else if (PPC::G8RCRegClass.contains(Reg) ||
1162 PPC::G8RC_NOX0RegClass.contains(Reg)) {
1163 OpcodeIndex = SOK_Int8Spill;
1164 } else if (PPC::F8RCRegClass.contains(Reg)) {
1165 OpcodeIndex = SOK_Float8Spill;
1166 } else if (PPC::F4RCRegClass.contains(Reg)) {
1167 OpcodeIndex = SOK_Float4Spill;
1168 } else if (PPC::SPERCRegClass.contains(Reg)) {
1169 OpcodeIndex = SOK_SPESpill;
1170 } else if (PPC::CRRCRegClass.contains(Reg)) {
1171 OpcodeIndex = SOK_CRSpill;
1172 } else if (PPC::CRBITRCRegClass.contains(Reg)) {
1173 OpcodeIndex = SOK_CRBitSpill;
1174 } else if (PPC::VRRCRegClass.contains(Reg)) {
1175 OpcodeIndex = SOK_VRVectorSpill;
1176 } else if (PPC::VSRCRegClass.contains(Reg)) {
1177 OpcodeIndex = SOK_VSXVectorSpill;
1178 } else if (PPC::VSFRCRegClass.contains(Reg)) {
1179 OpcodeIndex = SOK_VectorFloat8Spill;
1180 } else if (PPC::VSSRCRegClass.contains(Reg)) {
1181 OpcodeIndex = SOK_VectorFloat4Spill;
1182 } else if (PPC::VRSAVERCRegClass.contains(Reg)) {
1183 OpcodeIndex = SOK_VRSaveSpill;
1184 } else if (PPC::QFRCRegClass.contains(Reg)) {
1185 OpcodeIndex = SOK_QuadFloat8Spill;
1186 } else if (PPC::QSRCRegClass.contains(Reg)) {
1187 OpcodeIndex = SOK_QuadFloat4Spill;
1188 } else if (PPC::QBRCRegClass.contains(Reg)) {
1189 OpcodeIndex = SOK_QuadBitSpill;
1190 } else if (PPC::SPILLTOVSRRCRegClass.contains(Reg)) {
1191 OpcodeIndex = SOK_SpillToVSR;
1192 } else {
1193 llvm_unreachable("Unknown regclass!");
1194 }
1195 }
1196 return OpcodesForSpill[OpcodeIndex];
1197 }
1198
StoreRegToStackSlot(MachineFunction & MF,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1199 void PPCInstrInfo::StoreRegToStackSlot(
1200 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1201 const TargetRegisterClass *RC,
1202 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1203 unsigned Opcode = getStoreOpcodeForSpill(PPC::NoRegister, RC);
1204 DebugLoc DL;
1205
1206 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1207 FuncInfo->setHasSpills();
1208
1209 NewMIs.push_back(addFrameReference(
1210 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1211 FrameIdx));
1212
1213 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1214 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1215 FuncInfo->setSpillsCR();
1216
1217 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1218 FuncInfo->setSpillsVRSAVE();
1219
1220 if (isXFormMemOp(Opcode))
1221 FuncInfo->setHasNonRISpills();
1222 }
1223
storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1224 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1225 MachineBasicBlock::iterator MI,
1226 unsigned SrcReg, bool isKill,
1227 int FrameIdx,
1228 const TargetRegisterClass *RC,
1229 const TargetRegisterInfo *TRI) const {
1230 MachineFunction &MF = *MBB.getParent();
1231 SmallVector<MachineInstr *, 4> NewMIs;
1232
1233 // We need to avoid a situation in which the value from a VRRC register is
1234 // spilled using an Altivec instruction and reloaded into a VSRC register
1235 // using a VSX instruction. The issue with this is that the VSX
1236 // load/store instructions swap the doublewords in the vector and the Altivec
1237 // ones don't. The register classes on the spill/reload may be different if
1238 // the register is defined using an Altivec instruction and is then used by a
1239 // VSX instruction.
1240 RC = updatedRC(RC);
1241
1242 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1243
1244 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1245 MBB.insert(MI, NewMIs[i]);
1246
1247 const MachineFrameInfo &MFI = MF.getFrameInfo();
1248 MachineMemOperand *MMO = MF.getMachineMemOperand(
1249 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1250 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1251 MFI.getObjectAlignment(FrameIdx));
1252 NewMIs.back()->addMemOperand(MF, MMO);
1253 }
1254
LoadRegFromStackSlot(MachineFunction & MF,const DebugLoc & DL,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,SmallVectorImpl<MachineInstr * > & NewMIs) const1255 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1256 unsigned DestReg, int FrameIdx,
1257 const TargetRegisterClass *RC,
1258 SmallVectorImpl<MachineInstr *> &NewMIs)
1259 const {
1260 unsigned Opcode = getLoadOpcodeForSpill(PPC::NoRegister, RC);
1261 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1262 FrameIdx));
1263 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1264
1265 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1266 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1267 FuncInfo->setSpillsCR();
1268
1269 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1270 FuncInfo->setSpillsVRSAVE();
1271
1272 if (isXFormMemOp(Opcode))
1273 FuncInfo->setHasNonRISpills();
1274 }
1275
1276 void
loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,unsigned DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1277 PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1278 MachineBasicBlock::iterator MI,
1279 unsigned DestReg, int FrameIdx,
1280 const TargetRegisterClass *RC,
1281 const TargetRegisterInfo *TRI) const {
1282 MachineFunction &MF = *MBB.getParent();
1283 SmallVector<MachineInstr*, 4> NewMIs;
1284 DebugLoc DL;
1285 if (MI != MBB.end()) DL = MI->getDebugLoc();
1286
1287 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1288 FuncInfo->setHasSpills();
1289
1290 // We need to avoid a situation in which the value from a VRRC register is
1291 // spilled using an Altivec instruction and reloaded into a VSRC register
1292 // using a VSX instruction. The issue with this is that the VSX
1293 // load/store instructions swap the doublewords in the vector and the Altivec
1294 // ones don't. The register classes on the spill/reload may be different if
1295 // the register is defined using an Altivec instruction and is then used by a
1296 // VSX instruction.
1297 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
1298 RC = &PPC::VSRCRegClass;
1299
1300 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1301
1302 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1303 MBB.insert(MI, NewMIs[i]);
1304
1305 const MachineFrameInfo &MFI = MF.getFrameInfo();
1306 MachineMemOperand *MMO = MF.getMachineMemOperand(
1307 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1308 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1309 MFI.getObjectAlignment(FrameIdx));
1310 NewMIs.back()->addMemOperand(MF, MMO);
1311 }
1312
1313 bool PPCInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const1314 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
1315 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
1316 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
1317 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
1318 else
1319 // Leave the CR# the same, but invert the condition.
1320 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
1321 return false;
1322 }
1323
FoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,unsigned Reg,MachineRegisterInfo * MRI) const1324 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1325 unsigned Reg, MachineRegisterInfo *MRI) const {
1326 // For some instructions, it is legal to fold ZERO into the RA register field.
1327 // A zero immediate should always be loaded with a single li.
1328 unsigned DefOpc = DefMI.getOpcode();
1329 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
1330 return false;
1331 if (!DefMI.getOperand(1).isImm())
1332 return false;
1333 if (DefMI.getOperand(1).getImm() != 0)
1334 return false;
1335
1336 // Note that we cannot here invert the arguments of an isel in order to fold
1337 // a ZERO into what is presented as the second argument. All we have here
1338 // is the condition bit, and that might come from a CR-logical bit operation.
1339
1340 const MCInstrDesc &UseMCID = UseMI.getDesc();
1341
1342 // Only fold into real machine instructions.
1343 if (UseMCID.isPseudo())
1344 return false;
1345
1346 unsigned UseIdx;
1347 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
1348 if (UseMI.getOperand(UseIdx).isReg() &&
1349 UseMI.getOperand(UseIdx).getReg() == Reg)
1350 break;
1351
1352 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
1353 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
1354
1355 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
1356
1357 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
1358 // register (which might also be specified as a pointer class kind).
1359 if (UseInfo->isLookupPtrRegClass()) {
1360 if (UseInfo->RegClass /* Kind */ != 1)
1361 return false;
1362 } else {
1363 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
1364 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
1365 return false;
1366 }
1367
1368 // Make sure this is not tied to an output register (or otherwise
1369 // constrained). This is true for ST?UX registers, for example, which
1370 // are tied to their output registers.
1371 if (UseInfo->Constraints != 0)
1372 return false;
1373
1374 unsigned ZeroReg;
1375 if (UseInfo->isLookupPtrRegClass()) {
1376 bool isPPC64 = Subtarget.isPPC64();
1377 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
1378 } else {
1379 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
1380 PPC::ZERO8 : PPC::ZERO;
1381 }
1382
1383 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
1384 UseMI.getOperand(UseIdx).setReg(ZeroReg);
1385
1386 if (DeleteDef)
1387 DefMI.eraseFromParent();
1388
1389 return true;
1390 }
1391
MBBDefinesCTR(MachineBasicBlock & MBB)1392 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
1393 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
1394 I != IE; ++I)
1395 if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8))
1396 return true;
1397 return false;
1398 }
1399
1400 // We should make sure that, if we're going to predicate both sides of a
1401 // condition (a diamond), that both sides don't define the counter register. We
1402 // can predicate counter-decrement-based branches, but while that predicates
1403 // the branching, it does not predicate the counter decrement. If we tried to
1404 // merge the triangle into one predicated block, we'd decrement the counter
1405 // twice.
isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumT,unsigned ExtraT,MachineBasicBlock & FMBB,unsigned NumF,unsigned ExtraF,BranchProbability Probability) const1406 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
1407 unsigned NumT, unsigned ExtraT,
1408 MachineBasicBlock &FMBB,
1409 unsigned NumF, unsigned ExtraF,
1410 BranchProbability Probability) const {
1411 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
1412 }
1413
1414
isPredicated(const MachineInstr & MI) const1415 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
1416 // The predicated branches are identified by their type, not really by the
1417 // explicit presence of a predicate. Furthermore, some of them can be
1418 // predicated more than once. Because if conversion won't try to predicate
1419 // any instruction which already claims to be predicated (by returning true
1420 // here), always return false. In doing so, we let isPredicable() be the
1421 // final word on whether not the instruction can be (further) predicated.
1422
1423 return false;
1424 }
1425
isUnpredicatedTerminator(const MachineInstr & MI) const1426 bool PPCInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
1427 if (!MI.isTerminator())
1428 return false;
1429
1430 // Conditional branch is a special case.
1431 if (MI.isBranch() && !MI.isBarrier())
1432 return true;
1433
1434 return !isPredicated(MI);
1435 }
1436
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const1437 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
1438 ArrayRef<MachineOperand> Pred) const {
1439 unsigned OpC = MI.getOpcode();
1440 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
1441 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1442 bool isPPC64 = Subtarget.isPPC64();
1443 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
1444 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
1445 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1446 MI.setDesc(get(PPC::BCLR));
1447 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1448 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1449 MI.setDesc(get(PPC::BCLRn));
1450 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1451 } else {
1452 MI.setDesc(get(PPC::BCCLR));
1453 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1454 .addImm(Pred[0].getImm())
1455 .add(Pred[1]);
1456 }
1457
1458 return true;
1459 } else if (OpC == PPC::B) {
1460 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1461 bool isPPC64 = Subtarget.isPPC64();
1462 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
1463 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
1464 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1465 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1466 MI.RemoveOperand(0);
1467
1468 MI.setDesc(get(PPC::BC));
1469 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1470 .add(Pred[1])
1471 .addMBB(MBB);
1472 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1473 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1474 MI.RemoveOperand(0);
1475
1476 MI.setDesc(get(PPC::BCn));
1477 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1478 .add(Pred[1])
1479 .addMBB(MBB);
1480 } else {
1481 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1482 MI.RemoveOperand(0);
1483
1484 MI.setDesc(get(PPC::BCC));
1485 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1486 .addImm(Pred[0].getImm())
1487 .add(Pred[1])
1488 .addMBB(MBB);
1489 }
1490
1491 return true;
1492 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
1493 OpC == PPC::BCTRL8) {
1494 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
1495 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
1496
1497 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8;
1498 bool isPPC64 = Subtarget.isPPC64();
1499
1500 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1501 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
1502 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
1503 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1504 return true;
1505 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1506 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
1507 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
1508 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1509 return true;
1510 }
1511
1512 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
1513 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
1514 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1515 .addImm(Pred[0].getImm())
1516 .add(Pred[1]);
1517 return true;
1518 }
1519
1520 return false;
1521 }
1522
SubsumesPredicate(ArrayRef<MachineOperand> Pred1,ArrayRef<MachineOperand> Pred2) const1523 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
1524 ArrayRef<MachineOperand> Pred2) const {
1525 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
1526 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
1527
1528 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
1529 return false;
1530 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
1531 return false;
1532
1533 // P1 can only subsume P2 if they test the same condition register.
1534 if (Pred1[1].getReg() != Pred2[1].getReg())
1535 return false;
1536
1537 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
1538 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
1539
1540 if (P1 == P2)
1541 return true;
1542
1543 // Does P1 subsume P2, e.g. GE subsumes GT.
1544 if (P1 == PPC::PRED_LE &&
1545 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
1546 return true;
1547 if (P1 == PPC::PRED_GE &&
1548 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
1549 return true;
1550
1551 return false;
1552 }
1553
DefinesPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred) const1554 bool PPCInstrInfo::DefinesPredicate(MachineInstr &MI,
1555 std::vector<MachineOperand> &Pred) const {
1556 // Note: At the present time, the contents of Pred from this function is
1557 // unused by IfConversion. This implementation follows ARM by pushing the
1558 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
1559 // predicate, instructions defining CTR or CTR8 are also included as
1560 // predicate-defining instructions.
1561
1562 const TargetRegisterClass *RCs[] =
1563 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
1564 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
1565
1566 bool Found = false;
1567 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1568 const MachineOperand &MO = MI.getOperand(i);
1569 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
1570 const TargetRegisterClass *RC = RCs[c];
1571 if (MO.isReg()) {
1572 if (MO.isDef() && RC->contains(MO.getReg())) {
1573 Pred.push_back(MO);
1574 Found = true;
1575 }
1576 } else if (MO.isRegMask()) {
1577 for (TargetRegisterClass::iterator I = RC->begin(),
1578 IE = RC->end(); I != IE; ++I)
1579 if (MO.clobbersPhysReg(*I)) {
1580 Pred.push_back(MO);
1581 Found = true;
1582 }
1583 }
1584 }
1585 }
1586
1587 return Found;
1588 }
1589
analyzeCompare(const MachineInstr & MI,unsigned & SrcReg,unsigned & SrcReg2,int & Mask,int & Value) const1590 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
1591 unsigned &SrcReg2, int &Mask,
1592 int &Value) const {
1593 unsigned Opc = MI.getOpcode();
1594
1595 switch (Opc) {
1596 default: return false;
1597 case PPC::CMPWI:
1598 case PPC::CMPLWI:
1599 case PPC::CMPDI:
1600 case PPC::CMPLDI:
1601 SrcReg = MI.getOperand(1).getReg();
1602 SrcReg2 = 0;
1603 Value = MI.getOperand(2).getImm();
1604 Mask = 0xFFFF;
1605 return true;
1606 case PPC::CMPW:
1607 case PPC::CMPLW:
1608 case PPC::CMPD:
1609 case PPC::CMPLD:
1610 case PPC::FCMPUS:
1611 case PPC::FCMPUD:
1612 SrcReg = MI.getOperand(1).getReg();
1613 SrcReg2 = MI.getOperand(2).getReg();
1614 Value = 0;
1615 Mask = 0;
1616 return true;
1617 }
1618 }
1619
optimizeCompareInstr(MachineInstr & CmpInstr,unsigned SrcReg,unsigned SrcReg2,int Mask,int Value,const MachineRegisterInfo * MRI) const1620 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
1621 unsigned SrcReg2, int Mask, int Value,
1622 const MachineRegisterInfo *MRI) const {
1623 if (DisableCmpOpt)
1624 return false;
1625
1626 int OpC = CmpInstr.getOpcode();
1627 Register CRReg = CmpInstr.getOperand(0).getReg();
1628
1629 // FP record forms set CR1 based on the exception status bits, not a
1630 // comparison with zero.
1631 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
1632 return false;
1633
1634 const TargetRegisterInfo *TRI = &getRegisterInfo();
1635 // The record forms set the condition register based on a signed comparison
1636 // with zero (so says the ISA manual). This is not as straightforward as it
1637 // seems, however, because this is always a 64-bit comparison on PPC64, even
1638 // for instructions that are 32-bit in nature (like slw for example).
1639 // So, on PPC32, for unsigned comparisons, we can use the record forms only
1640 // for equality checks (as those don't depend on the sign). On PPC64,
1641 // we are restricted to equality for unsigned 64-bit comparisons and for
1642 // signed 32-bit comparisons the applicability is more restricted.
1643 bool isPPC64 = Subtarget.isPPC64();
1644 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
1645 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
1646 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
1647
1648 // Look through copies unless that gets us to a physical register.
1649 unsigned ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
1650 if (Register::isVirtualRegister(ActualSrc))
1651 SrcReg = ActualSrc;
1652
1653 // Get the unique definition of SrcReg.
1654 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1655 if (!MI) return false;
1656
1657 bool equalityOnly = false;
1658 bool noSub = false;
1659 if (isPPC64) {
1660 if (is32BitSignedCompare) {
1661 // We can perform this optimization only if MI is sign-extending.
1662 if (isSignExtended(*MI))
1663 noSub = true;
1664 else
1665 return false;
1666 } else if (is32BitUnsignedCompare) {
1667 // We can perform this optimization, equality only, if MI is
1668 // zero-extending.
1669 if (isZeroExtended(*MI)) {
1670 noSub = true;
1671 equalityOnly = true;
1672 } else
1673 return false;
1674 } else
1675 equalityOnly = is64BitUnsignedCompare;
1676 } else
1677 equalityOnly = is32BitUnsignedCompare;
1678
1679 if (equalityOnly) {
1680 // We need to check the uses of the condition register in order to reject
1681 // non-equality comparisons.
1682 for (MachineRegisterInfo::use_instr_iterator
1683 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
1684 I != IE; ++I) {
1685 MachineInstr *UseMI = &*I;
1686 if (UseMI->getOpcode() == PPC::BCC) {
1687 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1688 unsigned PredCond = PPC::getPredicateCondition(Pred);
1689 // We ignore hint bits when checking for non-equality comparisons.
1690 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
1691 return false;
1692 } else if (UseMI->getOpcode() == PPC::ISEL ||
1693 UseMI->getOpcode() == PPC::ISEL8) {
1694 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
1695 if (SubIdx != PPC::sub_eq)
1696 return false;
1697 } else
1698 return false;
1699 }
1700 }
1701
1702 MachineBasicBlock::iterator I = CmpInstr;
1703
1704 // Scan forward to find the first use of the compare.
1705 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
1706 ++I) {
1707 bool FoundUse = false;
1708 for (MachineRegisterInfo::use_instr_iterator
1709 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
1710 J != JE; ++J)
1711 if (&*J == &*I) {
1712 FoundUse = true;
1713 break;
1714 }
1715
1716 if (FoundUse)
1717 break;
1718 }
1719
1720 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
1721 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
1722
1723 // There are two possible candidates which can be changed to set CR[01].
1724 // One is MI, the other is a SUB instruction.
1725 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
1726 MachineInstr *Sub = nullptr;
1727 if (SrcReg2 != 0)
1728 // MI is not a candidate for CMPrr.
1729 MI = nullptr;
1730 // FIXME: Conservatively refuse to convert an instruction which isn't in the
1731 // same BB as the comparison. This is to allow the check below to avoid calls
1732 // (and other explicit clobbers); instead we should really check for these
1733 // more explicitly (in at least a few predecessors).
1734 else if (MI->getParent() != CmpInstr.getParent())
1735 return false;
1736 else if (Value != 0) {
1737 // The record-form instructions set CR bit based on signed comparison
1738 // against 0. We try to convert a compare against 1 or -1 into a compare
1739 // against 0 to exploit record-form instructions. For example, we change
1740 // the condition "greater than -1" into "greater than or equal to 0"
1741 // and "less than 1" into "less than or equal to 0".
1742
1743 // Since we optimize comparison based on a specific branch condition,
1744 // we don't optimize if condition code is used by more than once.
1745 if (equalityOnly || !MRI->hasOneUse(CRReg))
1746 return false;
1747
1748 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
1749 if (UseMI->getOpcode() != PPC::BCC)
1750 return false;
1751
1752 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1753 unsigned PredCond = PPC::getPredicateCondition(Pred);
1754 unsigned PredHint = PPC::getPredicateHint(Pred);
1755 int16_t Immed = (int16_t)Value;
1756
1757 // When modifying the condition in the predicate, we propagate hint bits
1758 // from the original predicate to the new one.
1759 if (Immed == -1 && PredCond == PPC::PRED_GT)
1760 // We convert "greater than -1" into "greater than or equal to 0",
1761 // since we are assuming signed comparison by !equalityOnly
1762 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
1763 else if (Immed == -1 && PredCond == PPC::PRED_LE)
1764 // We convert "less than or equal to -1" into "less than 0".
1765 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
1766 else if (Immed == 1 && PredCond == PPC::PRED_LT)
1767 // We convert "less than 1" into "less than or equal to 0".
1768 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
1769 else if (Immed == 1 && PredCond == PPC::PRED_GE)
1770 // We convert "greater than or equal to 1" into "greater than 0".
1771 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
1772 else
1773 return false;
1774
1775 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
1776 }
1777
1778 // Search for Sub.
1779 --I;
1780
1781 // Get ready to iterate backward from CmpInstr.
1782 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
1783
1784 for (; I != E && !noSub; --I) {
1785 const MachineInstr &Instr = *I;
1786 unsigned IOpC = Instr.getOpcode();
1787
1788 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
1789 Instr.readsRegister(PPC::CR0, TRI)))
1790 // This instruction modifies or uses the record condition register after
1791 // the one we want to change. While we could do this transformation, it
1792 // would likely not be profitable. This transformation removes one
1793 // instruction, and so even forcing RA to generate one move probably
1794 // makes it unprofitable.
1795 return false;
1796
1797 // Check whether CmpInstr can be made redundant by the current instruction.
1798 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
1799 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
1800 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
1801 ((Instr.getOperand(1).getReg() == SrcReg &&
1802 Instr.getOperand(2).getReg() == SrcReg2) ||
1803 (Instr.getOperand(1).getReg() == SrcReg2 &&
1804 Instr.getOperand(2).getReg() == SrcReg))) {
1805 Sub = &*I;
1806 break;
1807 }
1808
1809 if (I == B)
1810 // The 'and' is below the comparison instruction.
1811 return false;
1812 }
1813
1814 // Return false if no candidates exist.
1815 if (!MI && !Sub)
1816 return false;
1817
1818 // The single candidate is called MI.
1819 if (!MI) MI = Sub;
1820
1821 int NewOpC = -1;
1822 int MIOpC = MI->getOpcode();
1823 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
1824 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
1825 NewOpC = MIOpC;
1826 else {
1827 NewOpC = PPC::getRecordFormOpcode(MIOpC);
1828 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
1829 NewOpC = MIOpC;
1830 }
1831
1832 // FIXME: On the non-embedded POWER architectures, only some of the record
1833 // forms are fast, and we should use only the fast ones.
1834
1835 // The defining instruction has a record form (or is already a record
1836 // form). It is possible, however, that we'll need to reverse the condition
1837 // code of the users.
1838 if (NewOpC == -1)
1839 return false;
1840
1841 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
1842 // needs to be updated to be based on SUB. Push the condition code
1843 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
1844 // condition code of these operands will be modified.
1845 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
1846 // comparison against 0, which may modify predicate.
1847 bool ShouldSwap = false;
1848 if (Sub && Value == 0) {
1849 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
1850 Sub->getOperand(2).getReg() == SrcReg;
1851
1852 // The operands to subf are the opposite of sub, so only in the fixed-point
1853 // case, invert the order.
1854 ShouldSwap = !ShouldSwap;
1855 }
1856
1857 if (ShouldSwap)
1858 for (MachineRegisterInfo::use_instr_iterator
1859 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
1860 I != IE; ++I) {
1861 MachineInstr *UseMI = &*I;
1862 if (UseMI->getOpcode() == PPC::BCC) {
1863 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
1864 unsigned PredCond = PPC::getPredicateCondition(Pred);
1865 assert((!equalityOnly ||
1866 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
1867 "Invalid predicate for equality-only optimization");
1868 (void)PredCond; // To suppress warning in release build.
1869 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
1870 PPC::getSwappedPredicate(Pred)));
1871 } else if (UseMI->getOpcode() == PPC::ISEL ||
1872 UseMI->getOpcode() == PPC::ISEL8) {
1873 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
1874 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
1875 "Invalid CR bit for equality-only optimization");
1876
1877 if (NewSubReg == PPC::sub_lt)
1878 NewSubReg = PPC::sub_gt;
1879 else if (NewSubReg == PPC::sub_gt)
1880 NewSubReg = PPC::sub_lt;
1881
1882 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
1883 NewSubReg));
1884 } else // We need to abort on a user we don't understand.
1885 return false;
1886 }
1887 assert(!(Value != 0 && ShouldSwap) &&
1888 "Non-zero immediate support and ShouldSwap"
1889 "may conflict in updating predicate");
1890
1891 // Create a new virtual register to hold the value of the CR set by the
1892 // record-form instruction. If the instruction was not previously in
1893 // record form, then set the kill flag on the CR.
1894 CmpInstr.eraseFromParent();
1895
1896 MachineBasicBlock::iterator MII = MI;
1897 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
1898 get(TargetOpcode::COPY), CRReg)
1899 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
1900
1901 // Even if CR0 register were dead before, it is alive now since the
1902 // instruction we just built uses it.
1903 MI->clearRegisterDeads(PPC::CR0);
1904
1905 if (MIOpC != NewOpC) {
1906 // We need to be careful here: we're replacing one instruction with
1907 // another, and we need to make sure that we get all of the right
1908 // implicit uses and defs. On the other hand, the caller may be holding
1909 // an iterator to this instruction, and so we can't delete it (this is
1910 // specifically the case if this is the instruction directly after the
1911 // compare).
1912
1913 // Rotates are expensive instructions. If we're emitting a record-form
1914 // rotate that can just be an andi/andis, we should just emit that.
1915 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
1916 Register GPRRes = MI->getOperand(0).getReg();
1917 int64_t SH = MI->getOperand(2).getImm();
1918 int64_t MB = MI->getOperand(3).getImm();
1919 int64_t ME = MI->getOperand(4).getImm();
1920 // We can only do this if both the start and end of the mask are in the
1921 // same halfword.
1922 bool MBInLoHWord = MB >= 16;
1923 bool MEInLoHWord = ME >= 16;
1924 uint64_t Mask = ~0LLU;
1925
1926 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
1927 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
1928 // The mask value needs to shift right 16 if we're emitting andis.
1929 Mask >>= MBInLoHWord ? 0 : 16;
1930 NewOpC = MIOpC == PPC::RLWINM
1931 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
1932 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
1933 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
1934 (ME - MB + 1 == SH) && (MB >= 16)) {
1935 // If we are rotating by the exact number of bits as are in the mask
1936 // and the mask is in the least significant bits of the register,
1937 // that's just an andis. (as long as the GPR result has no uses).
1938 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
1939 Mask >>= 16;
1940 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
1941 }
1942 // If we've set the mask, we can transform.
1943 if (Mask != ~0LLU) {
1944 MI->RemoveOperand(4);
1945 MI->RemoveOperand(3);
1946 MI->getOperand(2).setImm(Mask);
1947 NumRcRotatesConvertedToRcAnd++;
1948 }
1949 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
1950 int64_t MB = MI->getOperand(3).getImm();
1951 if (MB >= 48) {
1952 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
1953 NewOpC = PPC::ANDI8_rec;
1954 MI->RemoveOperand(3);
1955 MI->getOperand(2).setImm(Mask);
1956 NumRcRotatesConvertedToRcAnd++;
1957 }
1958 }
1959
1960 const MCInstrDesc &NewDesc = get(NewOpC);
1961 MI->setDesc(NewDesc);
1962
1963 if (NewDesc.ImplicitDefs)
1964 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
1965 *ImpDefs; ++ImpDefs)
1966 if (!MI->definesRegister(*ImpDefs))
1967 MI->addOperand(*MI->getParent()->getParent(),
1968 MachineOperand::CreateReg(*ImpDefs, true, true));
1969 if (NewDesc.ImplicitUses)
1970 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
1971 *ImpUses; ++ImpUses)
1972 if (!MI->readsRegister(*ImpUses))
1973 MI->addOperand(*MI->getParent()->getParent(),
1974 MachineOperand::CreateReg(*ImpUses, false, true));
1975 }
1976 assert(MI->definesRegister(PPC::CR0) &&
1977 "Record-form instruction does not define cr0?");
1978
1979 // Modify the condition code of operands in OperandsToUpdate.
1980 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
1981 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
1982 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
1983 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
1984
1985 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
1986 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
1987
1988 return true;
1989 }
1990
1991 /// GetInstSize - Return the number of bytes of code the specified
1992 /// instruction may be. This returns the maximum number of bytes.
1993 ///
getInstSizeInBytes(const MachineInstr & MI) const1994 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1995 unsigned Opcode = MI.getOpcode();
1996
1997 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
1998 const MachineFunction *MF = MI.getParent()->getParent();
1999 const char *AsmStr = MI.getOperand(0).getSymbolName();
2000 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2001 } else if (Opcode == TargetOpcode::STACKMAP) {
2002 StackMapOpers Opers(&MI);
2003 return Opers.getNumPatchBytes();
2004 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2005 PatchPointOpers Opers(&MI);
2006 return Opers.getNumPatchBytes();
2007 } else {
2008 return get(Opcode).getSize();
2009 }
2010 }
2011
2012 std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const2013 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2014 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2015 return std::make_pair(TF & Mask, TF & ~Mask);
2016 }
2017
2018 ArrayRef<std::pair<unsigned, const char *>>
getSerializableDirectMachineOperandTargetFlags() const2019 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2020 using namespace PPCII;
2021 static const std::pair<unsigned, const char *> TargetFlags[] = {
2022 {MO_LO, "ppc-lo"},
2023 {MO_HA, "ppc-ha"},
2024 {MO_TPREL_LO, "ppc-tprel-lo"},
2025 {MO_TPREL_HA, "ppc-tprel-ha"},
2026 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2027 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2028 {MO_TOC_LO, "ppc-toc-lo"},
2029 {MO_TLS, "ppc-tls"}};
2030 return makeArrayRef(TargetFlags);
2031 }
2032
2033 ArrayRef<std::pair<unsigned, const char *>>
getSerializableBitmaskMachineOperandTargetFlags() const2034 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
2035 using namespace PPCII;
2036 static const std::pair<unsigned, const char *> TargetFlags[] = {
2037 {MO_PLT, "ppc-plt"},
2038 {MO_PIC_FLAG, "ppc-pic"},
2039 {MO_NLP_FLAG, "ppc-nlp"},
2040 {MO_NLP_HIDDEN_FLAG, "ppc-nlp-hidden"}};
2041 return makeArrayRef(TargetFlags);
2042 }
2043
2044 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2045 // The VSX versions have the advantage of a full 64-register target whereas
2046 // the FP ones have the advantage of lower latency and higher throughput. So
2047 // what we are after is using the faster instructions in low register pressure
2048 // situations and using the larger register file in high register pressure
2049 // situations.
expandVSXMemPseudo(MachineInstr & MI) const2050 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2051 unsigned UpperOpcode, LowerOpcode;
2052 switch (MI.getOpcode()) {
2053 case PPC::DFLOADf32:
2054 UpperOpcode = PPC::LXSSP;
2055 LowerOpcode = PPC::LFS;
2056 break;
2057 case PPC::DFLOADf64:
2058 UpperOpcode = PPC::LXSD;
2059 LowerOpcode = PPC::LFD;
2060 break;
2061 case PPC::DFSTOREf32:
2062 UpperOpcode = PPC::STXSSP;
2063 LowerOpcode = PPC::STFS;
2064 break;
2065 case PPC::DFSTOREf64:
2066 UpperOpcode = PPC::STXSD;
2067 LowerOpcode = PPC::STFD;
2068 break;
2069 case PPC::XFLOADf32:
2070 UpperOpcode = PPC::LXSSPX;
2071 LowerOpcode = PPC::LFSX;
2072 break;
2073 case PPC::XFLOADf64:
2074 UpperOpcode = PPC::LXSDX;
2075 LowerOpcode = PPC::LFDX;
2076 break;
2077 case PPC::XFSTOREf32:
2078 UpperOpcode = PPC::STXSSPX;
2079 LowerOpcode = PPC::STFSX;
2080 break;
2081 case PPC::XFSTOREf64:
2082 UpperOpcode = PPC::STXSDX;
2083 LowerOpcode = PPC::STFDX;
2084 break;
2085 case PPC::LIWAX:
2086 UpperOpcode = PPC::LXSIWAX;
2087 LowerOpcode = PPC::LFIWAX;
2088 break;
2089 case PPC::LIWZX:
2090 UpperOpcode = PPC::LXSIWZX;
2091 LowerOpcode = PPC::LFIWZX;
2092 break;
2093 case PPC::STIWX:
2094 UpperOpcode = PPC::STXSIWX;
2095 LowerOpcode = PPC::STFIWX;
2096 break;
2097 default:
2098 llvm_unreachable("Unknown Operation!");
2099 }
2100
2101 Register TargetReg = MI.getOperand(0).getReg();
2102 unsigned Opcode;
2103 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2104 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2105 Opcode = LowerOpcode;
2106 else
2107 Opcode = UpperOpcode;
2108 MI.setDesc(get(Opcode));
2109 return true;
2110 }
2111
isAnImmediateOperand(const MachineOperand & MO)2112 static bool isAnImmediateOperand(const MachineOperand &MO) {
2113 return MO.isCPI() || MO.isGlobal() || MO.isImm();
2114 }
2115
expandPostRAPseudo(MachineInstr & MI) const2116 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2117 auto &MBB = *MI.getParent();
2118 auto DL = MI.getDebugLoc();
2119
2120 switch (MI.getOpcode()) {
2121 case TargetOpcode::LOAD_STACK_GUARD: {
2122 assert(Subtarget.isTargetLinux() &&
2123 "Only Linux target is expected to contain LOAD_STACK_GUARD");
2124 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
2125 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
2126 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
2127 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2128 .addImm(Offset)
2129 .addReg(Reg);
2130 return true;
2131 }
2132 case PPC::DFLOADf32:
2133 case PPC::DFLOADf64:
2134 case PPC::DFSTOREf32:
2135 case PPC::DFSTOREf64: {
2136 assert(Subtarget.hasP9Vector() &&
2137 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
2138 assert(MI.getOperand(2).isReg() &&
2139 isAnImmediateOperand(MI.getOperand(1)) &&
2140 "D-form op must have register and immediate operands");
2141 return expandVSXMemPseudo(MI);
2142 }
2143 case PPC::XFLOADf32:
2144 case PPC::XFSTOREf32:
2145 case PPC::LIWAX:
2146 case PPC::LIWZX:
2147 case PPC::STIWX: {
2148 assert(Subtarget.hasP8Vector() &&
2149 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
2150 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2151 "X-form op must have register and register operands");
2152 return expandVSXMemPseudo(MI);
2153 }
2154 case PPC::XFLOADf64:
2155 case PPC::XFSTOREf64: {
2156 assert(Subtarget.hasVSX() &&
2157 "Invalid X-Form Pseudo-ops on target that has no VSX.");
2158 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2159 "X-form op must have register and register operands");
2160 return expandVSXMemPseudo(MI);
2161 }
2162 case PPC::SPILLTOVSR_LD: {
2163 Register TargetReg = MI.getOperand(0).getReg();
2164 if (PPC::VSFRCRegClass.contains(TargetReg)) {
2165 MI.setDesc(get(PPC::DFLOADf64));
2166 return expandPostRAPseudo(MI);
2167 }
2168 else
2169 MI.setDesc(get(PPC::LD));
2170 return true;
2171 }
2172 case PPC::SPILLTOVSR_ST: {
2173 Register SrcReg = MI.getOperand(0).getReg();
2174 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2175 NumStoreSPILLVSRRCAsVec++;
2176 MI.setDesc(get(PPC::DFSTOREf64));
2177 return expandPostRAPseudo(MI);
2178 } else {
2179 NumStoreSPILLVSRRCAsGpr++;
2180 MI.setDesc(get(PPC::STD));
2181 }
2182 return true;
2183 }
2184 case PPC::SPILLTOVSR_LDX: {
2185 Register TargetReg = MI.getOperand(0).getReg();
2186 if (PPC::VSFRCRegClass.contains(TargetReg))
2187 MI.setDesc(get(PPC::LXSDX));
2188 else
2189 MI.setDesc(get(PPC::LDX));
2190 return true;
2191 }
2192 case PPC::SPILLTOVSR_STX: {
2193 Register SrcReg = MI.getOperand(0).getReg();
2194 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2195 NumStoreSPILLVSRRCAsVec++;
2196 MI.setDesc(get(PPC::STXSDX));
2197 } else {
2198 NumStoreSPILLVSRRCAsGpr++;
2199 MI.setDesc(get(PPC::STDX));
2200 }
2201 return true;
2202 }
2203
2204 case PPC::CFENCE8: {
2205 auto Val = MI.getOperand(0).getReg();
2206 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
2207 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
2208 .addImm(PPC::PRED_NE_MINUS)
2209 .addReg(PPC::CR7)
2210 .addImm(1);
2211 MI.setDesc(get(PPC::ISYNC));
2212 MI.RemoveOperand(0);
2213 return true;
2214 }
2215 }
2216 return false;
2217 }
2218
2219 // Essentially a compile-time implementation of a compare->isel sequence.
2220 // It takes two constants to compare, along with the true/false registers
2221 // and the comparison type (as a subreg to a CR field) and returns one
2222 // of the true/false registers, depending on the comparison results.
selectReg(int64_t Imm1,int64_t Imm2,unsigned CompareOpc,unsigned TrueReg,unsigned FalseReg,unsigned CRSubReg)2223 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
2224 unsigned TrueReg, unsigned FalseReg,
2225 unsigned CRSubReg) {
2226 // Signed comparisons. The immediates are assumed to be sign-extended.
2227 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
2228 switch (CRSubReg) {
2229 default: llvm_unreachable("Unknown integer comparison type.");
2230 case PPC::sub_lt:
2231 return Imm1 < Imm2 ? TrueReg : FalseReg;
2232 case PPC::sub_gt:
2233 return Imm1 > Imm2 ? TrueReg : FalseReg;
2234 case PPC::sub_eq:
2235 return Imm1 == Imm2 ? TrueReg : FalseReg;
2236 }
2237 }
2238 // Unsigned comparisons.
2239 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
2240 switch (CRSubReg) {
2241 default: llvm_unreachable("Unknown integer comparison type.");
2242 case PPC::sub_lt:
2243 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
2244 case PPC::sub_gt:
2245 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
2246 case PPC::sub_eq:
2247 return Imm1 == Imm2 ? TrueReg : FalseReg;
2248 }
2249 }
2250 return PPC::NoRegister;
2251 }
2252
replaceInstrOperandWithImm(MachineInstr & MI,unsigned OpNo,int64_t Imm) const2253 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
2254 unsigned OpNo,
2255 int64_t Imm) const {
2256 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
2257 // Replace the REG with the Immediate.
2258 Register InUseReg = MI.getOperand(OpNo).getReg();
2259 MI.getOperand(OpNo).ChangeToImmediate(Imm);
2260
2261 if (MI.implicit_operands().empty())
2262 return;
2263
2264 // We need to make sure that the MI didn't have any implicit use
2265 // of this REG any more.
2266 const TargetRegisterInfo *TRI = &getRegisterInfo();
2267 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
2268 if (UseOpIdx >= 0) {
2269 MachineOperand &MO = MI.getOperand(UseOpIdx);
2270 if (MO.isImplicit())
2271 // The operands must always be in the following order:
2272 // - explicit reg defs,
2273 // - other explicit operands (reg uses, immediates, etc.),
2274 // - implicit reg defs
2275 // - implicit reg uses
2276 // Therefore, removing the implicit operand won't change the explicit
2277 // operands layout.
2278 MI.RemoveOperand(UseOpIdx);
2279 }
2280 }
2281
2282 // Replace an instruction with one that materializes a constant (and sets
2283 // CR0 if the original instruction was a record-form instruction).
replaceInstrWithLI(MachineInstr & MI,const LoadImmediateInfo & LII) const2284 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
2285 const LoadImmediateInfo &LII) const {
2286 // Remove existing operands.
2287 int OperandToKeep = LII.SetCR ? 1 : 0;
2288 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
2289 MI.RemoveOperand(i);
2290
2291 // Replace the instruction.
2292 if (LII.SetCR) {
2293 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
2294 // Set the immediate.
2295 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2296 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
2297 return;
2298 }
2299 else
2300 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
2301
2302 // Set the immediate.
2303 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2304 .addImm(LII.Imm);
2305 }
2306
getDefMIPostRA(unsigned Reg,MachineInstr & MI,bool & SeenIntermediateUse) const2307 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
2308 bool &SeenIntermediateUse) const {
2309 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
2310 "Should be called after register allocation.");
2311 const TargetRegisterInfo *TRI = &getRegisterInfo();
2312 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
2313 It++;
2314 SeenIntermediateUse = false;
2315 for (; It != E; ++It) {
2316 if (It->modifiesRegister(Reg, TRI))
2317 return &*It;
2318 if (It->readsRegister(Reg, TRI))
2319 SeenIntermediateUse = true;
2320 }
2321 return nullptr;
2322 }
2323
getForwardingDefMI(MachineInstr & MI,unsigned & OpNoForForwarding,bool & SeenIntermediateUse) const2324 MachineInstr *PPCInstrInfo::getForwardingDefMI(
2325 MachineInstr &MI,
2326 unsigned &OpNoForForwarding,
2327 bool &SeenIntermediateUse) const {
2328 OpNoForForwarding = ~0U;
2329 MachineInstr *DefMI = nullptr;
2330 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
2331 const TargetRegisterInfo *TRI = &getRegisterInfo();
2332 // If we're in SSA, get the defs through the MRI. Otherwise, only look
2333 // within the basic block to see if the register is defined using an LI/LI8.
2334 if (MRI->isSSA()) {
2335 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2336 if (!MI.getOperand(i).isReg())
2337 continue;
2338 Register Reg = MI.getOperand(i).getReg();
2339 if (!Register::isVirtualRegister(Reg))
2340 continue;
2341 unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
2342 if (Register::isVirtualRegister(TrueReg)) {
2343 DefMI = MRI->getVRegDef(TrueReg);
2344 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8) {
2345 OpNoForForwarding = i;
2346 break;
2347 }
2348 }
2349 }
2350 } else {
2351 // Looking back through the definition for each operand could be expensive,
2352 // so exit early if this isn't an instruction that either has an immediate
2353 // form or is already an immediate form that we can handle.
2354 ImmInstrInfo III;
2355 unsigned Opc = MI.getOpcode();
2356 bool ConvertibleImmForm =
2357 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
2358 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
2359 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
2360 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
2361 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
2362 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
2363 Opc == PPC::RLWINM8_rec;
2364 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
2365 ? isVFRegister(MI.getOperand(0).getReg())
2366 : false;
2367 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
2368 return nullptr;
2369
2370 // Don't convert or %X, %Y, %Y since that's just a register move.
2371 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
2372 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2373 return nullptr;
2374 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2375 MachineOperand &MO = MI.getOperand(i);
2376 SeenIntermediateUse = false;
2377 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
2378 Register Reg = MI.getOperand(i).getReg();
2379 // If we see another use of this reg between the def and the MI,
2380 // we want to flat it so the def isn't deleted.
2381 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
2382 if (DefMI) {
2383 // Is this register defined by some form of add-immediate (including
2384 // load-immediate) within this basic block?
2385 switch (DefMI->getOpcode()) {
2386 default:
2387 break;
2388 case PPC::LI:
2389 case PPC::LI8:
2390 case PPC::ADDItocL:
2391 case PPC::ADDI:
2392 case PPC::ADDI8:
2393 OpNoForForwarding = i;
2394 return DefMI;
2395 }
2396 }
2397 }
2398 }
2399 }
2400 return OpNoForForwarding == ~0U ? nullptr : DefMI;
2401 }
2402
getStoreOpcodesForSpillArray() const2403 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
2404 static const unsigned OpcodesForSpill[2][SOK_LastOpcodeSpill] = {
2405 // Power 8
2406 {PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR,
2407 PPC::SPILL_CRBIT, PPC::STVX, PPC::STXVD2X, PPC::STXSDX, PPC::STXSSPX,
2408 PPC::SPILL_VRSAVE, PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb,
2409 PPC::SPILLTOVSR_ST, PPC::EVSTDD},
2410 // Power 9
2411 {PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR,
2412 PPC::SPILL_CRBIT, PPC::STVX, PPC::STXV, PPC::DFSTOREf64, PPC::DFSTOREf32,
2413 PPC::SPILL_VRSAVE, PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb,
2414 PPC::SPILLTOVSR_ST}};
2415
2416 return OpcodesForSpill[(Subtarget.hasP9Vector()) ? 1 : 0];
2417 }
2418
getLoadOpcodesForSpillArray() const2419 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
2420 static const unsigned OpcodesForSpill[2][SOK_LastOpcodeSpill] = {
2421 // Power 8
2422 {PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,
2423 PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXVD2X, PPC::LXSDX, PPC::LXSSPX,
2424 PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs, PPC::QVLFDXb,
2425 PPC::SPILLTOVSR_LD, PPC::EVLDD},
2426 // Power 9
2427 {PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,
2428 PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXV, PPC::DFLOADf64, PPC::DFLOADf32,
2429 PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs, PPC::QVLFDXb,
2430 PPC::SPILLTOVSR_LD}};
2431
2432 return OpcodesForSpill[(Subtarget.hasP9Vector()) ? 1 : 0];
2433 }
2434
fixupIsDeadOrKill(MachineInstr & StartMI,MachineInstr & EndMI,unsigned RegNo) const2435 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr &StartMI, MachineInstr &EndMI,
2436 unsigned RegNo) const {
2437 const MachineRegisterInfo &MRI =
2438 StartMI.getParent()->getParent()->getRegInfo();
2439 if (MRI.isSSA())
2440 return;
2441
2442 // Instructions between [StartMI, EndMI] should be in same basic block.
2443 assert((StartMI.getParent() == EndMI.getParent()) &&
2444 "Instructions are not in same basic block");
2445
2446 bool IsKillSet = false;
2447
2448 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
2449 MachineOperand &MO = MI.getOperand(Index);
2450 if (MO.isReg() && MO.isUse() && MO.isKill() &&
2451 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
2452 MO.setIsKill(false);
2453 };
2454
2455 // Set killed flag for EndMI.
2456 // No need to do anything if EndMI defines RegNo.
2457 int UseIndex =
2458 EndMI.findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
2459 if (UseIndex != -1) {
2460 EndMI.getOperand(UseIndex).setIsKill(true);
2461 IsKillSet = true;
2462 // Clear killed flag for other EndMI operands related to RegNo. In some
2463 // upexpected cases, killed may be set multiple times for same register
2464 // operand in same MI.
2465 for (int i = 0, e = EndMI.getNumOperands(); i != e; ++i)
2466 if (i != UseIndex)
2467 clearOperandKillInfo(EndMI, i);
2468 }
2469
2470 // Walking the inst in reverse order (EndMI -> StartMI].
2471 MachineBasicBlock::reverse_iterator It = EndMI;
2472 MachineBasicBlock::reverse_iterator E = EndMI.getParent()->rend();
2473 // EndMI has been handled above, skip it here.
2474 It++;
2475 MachineOperand *MO = nullptr;
2476 for (; It != E; ++It) {
2477 // Skip insturctions which could not be a def/use of RegNo.
2478 if (It->isDebugInstr() || It->isPosition())
2479 continue;
2480
2481 // Clear killed flag for all It operands related to RegNo. In some
2482 // upexpected cases, killed may be set multiple times for same register
2483 // operand in same MI.
2484 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
2485 clearOperandKillInfo(*It, i);
2486
2487 // If killed is not set, set killed for its last use or set dead for its def
2488 // if no use found.
2489 if (!IsKillSet) {
2490 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
2491 // Use found, set it killed.
2492 IsKillSet = true;
2493 MO->setIsKill(true);
2494 continue;
2495 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
2496 &getRegisterInfo()))) {
2497 // No use found, set dead for its def.
2498 assert(&*It == &StartMI && "No new def between StartMI and EndMI.");
2499 MO->setIsDead(true);
2500 break;
2501 }
2502 }
2503
2504 if ((&*It) == &StartMI)
2505 break;
2506 }
2507 // Ensure RegMo liveness is killed after EndMI.
2508 assert((IsKillSet || (MO && MO->isDead())) &&
2509 "RegNo should be killed or dead");
2510 }
2511
2512 // This opt tries to convert the following imm form to an index form to save an
2513 // add for stack variables.
2514 // Return false if no such pattern found.
2515 //
2516 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2517 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
2518 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
2519 //
2520 // can be converted to:
2521 //
2522 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
2523 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
2524 //
2525 // In order to eliminate ADD instr, make sure that:
2526 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
2527 // new ADDI instr and ADDI can only take int16 Imm.
2528 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
2529 // between ADDI and ADD instr since its original def in ADDI will be changed
2530 // in new ADDI instr. And also there should be no new def for it between
2531 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
2532 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
2533 // between ADD and Imm instr since ADD instr will be eliminated.
2534 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
2535 // moved to Index instr.
foldFrameOffset(MachineInstr & MI) const2536 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
2537 MachineFunction *MF = MI.getParent()->getParent();
2538 MachineRegisterInfo *MRI = &MF->getRegInfo();
2539 bool PostRA = !MRI->isSSA();
2540 // Do this opt after PEI which is after RA. The reason is stack slot expansion
2541 // in PEI may expose such opportunities since in PEI, stack slot offsets to
2542 // frame base(OffsetAddi) are determined.
2543 if (!PostRA)
2544 return false;
2545 unsigned ToBeDeletedReg = 0;
2546 int64_t OffsetImm = 0;
2547 unsigned XFormOpcode = 0;
2548 ImmInstrInfo III;
2549
2550 // Check if Imm instr meets requirement.
2551 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
2552 III))
2553 return false;
2554
2555 bool OtherIntermediateUse = false;
2556 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
2557
2558 // Exit if there is other use between ADD and Imm instr or no def found.
2559 if (OtherIntermediateUse || !ADDMI)
2560 return false;
2561
2562 // Check if ADD instr meets requirement.
2563 if (!isADDInstrEligibleForFolding(*ADDMI))
2564 return false;
2565
2566 unsigned ScaleRegIdx = 0;
2567 int64_t OffsetAddi = 0;
2568 MachineInstr *ADDIMI = nullptr;
2569
2570 // Check if there is a valid ToBeChangedReg in ADDMI.
2571 // 1: It must be killed.
2572 // 2: Its definition must be a valid ADDIMI.
2573 // 3: It must satify int16 offset requirement.
2574 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
2575 ScaleRegIdx = 2;
2576 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
2577 ScaleRegIdx = 1;
2578 else
2579 return false;
2580
2581 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
2582 unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg();
2583 unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
2584 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
2585 MachineBasicBlock::iterator End) {
2586 for (auto It = ++Start; It != End; It++)
2587 if (It->modifiesRegister(Reg, &getRegisterInfo()))
2588 return true;
2589 return false;
2590 };
2591 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
2592 // and Imm Instr.
2593 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
2594 return false;
2595
2596 // Now start to do the transformation.
2597 LLVM_DEBUG(dbgs() << "Replace instruction: "
2598 << "\n");
2599 LLVM_DEBUG(ADDIMI->dump());
2600 LLVM_DEBUG(ADDMI->dump());
2601 LLVM_DEBUG(MI.dump());
2602 LLVM_DEBUG(dbgs() << "with: "
2603 << "\n");
2604
2605 // Update ADDI instr.
2606 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
2607
2608 // Update Imm instr.
2609 MI.setDesc(get(XFormOpcode));
2610 MI.getOperand(III.ImmOpNo)
2611 .ChangeToRegister(ScaleReg, false, false,
2612 ADDMI->getOperand(ScaleRegIdx).isKill());
2613
2614 MI.getOperand(III.OpNoForForwarding)
2615 .ChangeToRegister(ToBeChangedReg, false, false, true);
2616
2617 // Eliminate ADD instr.
2618 ADDMI->eraseFromParent();
2619
2620 LLVM_DEBUG(ADDIMI->dump());
2621 LLVM_DEBUG(MI.dump());
2622
2623 return true;
2624 }
2625
isADDIInstrEligibleForFolding(MachineInstr & ADDIMI,int64_t & Imm) const2626 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
2627 int64_t &Imm) const {
2628 unsigned Opc = ADDIMI.getOpcode();
2629
2630 // Exit if the instruction is not ADDI.
2631 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
2632 return false;
2633
2634 Imm = ADDIMI.getOperand(2).getImm();
2635
2636 return true;
2637 }
2638
isADDInstrEligibleForFolding(MachineInstr & ADDMI) const2639 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
2640 unsigned Opc = ADDMI.getOpcode();
2641
2642 // Exit if the instruction is not ADD.
2643 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
2644 }
2645
isImmInstrEligibleForFolding(MachineInstr & MI,unsigned & ToBeDeletedReg,unsigned & XFormOpcode,int64_t & OffsetImm,ImmInstrInfo & III) const2646 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
2647 unsigned &ToBeDeletedReg,
2648 unsigned &XFormOpcode,
2649 int64_t &OffsetImm,
2650 ImmInstrInfo &III) const {
2651 // Only handle load/store.
2652 if (!MI.mayLoadOrStore())
2653 return false;
2654
2655 unsigned Opc = MI.getOpcode();
2656
2657 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
2658
2659 // Exit if instruction has no index form.
2660 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
2661 return false;
2662
2663 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
2664 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
2665 III, true))
2666 return false;
2667
2668 if (!III.IsSummingOperands)
2669 return false;
2670
2671 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
2672 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
2673 // Only support imm operands, not relocation slots or others.
2674 if (!ImmOperand.isImm())
2675 return false;
2676
2677 assert(RegOperand.isReg() && "Instruction format is not right");
2678
2679 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
2680 if (!RegOperand.isKill())
2681 return false;
2682
2683 ToBeDeletedReg = RegOperand.getReg();
2684 OffsetImm = ImmOperand.getImm();
2685
2686 return true;
2687 }
2688
isValidToBeChangedReg(MachineInstr * ADDMI,unsigned Index,MachineInstr * & ADDIMI,int64_t & OffsetAddi,int64_t OffsetImm) const2689 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
2690 MachineInstr *&ADDIMI,
2691 int64_t &OffsetAddi,
2692 int64_t OffsetImm) const {
2693 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
2694 MachineOperand &MO = ADDMI->getOperand(Index);
2695
2696 if (!MO.isKill())
2697 return false;
2698
2699 bool OtherIntermediateUse = false;
2700
2701 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
2702 // Currently handle only one "add + Imminstr" pair case, exit if other
2703 // intermediate use for ToBeChangedReg found.
2704 // TODO: handle the cases where there are other "add + Imminstr" pairs
2705 // with same offset in Imminstr which is like:
2706 //
2707 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2708 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
2709 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
2710 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
2711 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
2712 //
2713 // can be converted to:
2714 //
2715 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
2716 // (OffsetAddi + OffsetImm)
2717 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
2718 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
2719
2720 if (OtherIntermediateUse || !ADDIMI)
2721 return false;
2722 // Check if ADDI instr meets requirement.
2723 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
2724 return false;
2725
2726 if (isInt<16>(OffsetAddi + OffsetImm))
2727 return true;
2728 return false;
2729 }
2730
2731 // If this instruction has an immediate form and one of its operands is a
2732 // result of a load-immediate or an add-immediate, convert it to
2733 // the immediate form if the constant is in range.
convertToImmediateForm(MachineInstr & MI,MachineInstr ** KilledDef) const2734 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
2735 MachineInstr **KilledDef) const {
2736 MachineFunction *MF = MI.getParent()->getParent();
2737 MachineRegisterInfo *MRI = &MF->getRegInfo();
2738 bool PostRA = !MRI->isSSA();
2739 bool SeenIntermediateUse = true;
2740 unsigned ForwardingOperand = ~0U;
2741 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
2742 SeenIntermediateUse);
2743 if (!DefMI)
2744 return false;
2745 assert(ForwardingOperand < MI.getNumOperands() &&
2746 "The forwarding operand needs to be valid at this point");
2747 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
2748 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
2749 Register ForwardingOperandReg = MI.getOperand(ForwardingOperand).getReg();
2750 if (KilledDef && KillFwdDefMI)
2751 *KilledDef = DefMI;
2752
2753 ImmInstrInfo III;
2754 bool IsVFReg = MI.getOperand(0).isReg()
2755 ? isVFRegister(MI.getOperand(0).getReg())
2756 : false;
2757 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
2758 // If this is a reg+reg instruction that has a reg+imm form,
2759 // and one of the operands is produced by an add-immediate,
2760 // try to convert it.
2761 if (HasImmForm &&
2762 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
2763 KillFwdDefMI))
2764 return true;
2765
2766 if ((DefMI->getOpcode() != PPC::LI && DefMI->getOpcode() != PPC::LI8) ||
2767 !DefMI->getOperand(1).isImm())
2768 return false;
2769
2770 int64_t Immediate = DefMI->getOperand(1).getImm();
2771 // Sign-extend to 64-bits.
2772 int64_t SExtImm = ((uint64_t)Immediate & ~0x7FFFuLL) != 0 ?
2773 (Immediate | 0xFFFFFFFFFFFF0000) : Immediate;
2774
2775 // If this is a reg+reg instruction that has a reg+imm form,
2776 // and one of the operands is produced by LI, convert it now.
2777 if (HasImmForm)
2778 return transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI, SExtImm);
2779
2780 bool ReplaceWithLI = false;
2781 bool Is64BitLI = false;
2782 int64_t NewImm = 0;
2783 bool SetCR = false;
2784 unsigned Opc = MI.getOpcode();
2785 switch (Opc) {
2786 default: return false;
2787
2788 // FIXME: Any branches conditional on such a comparison can be made
2789 // unconditional. At this time, this happens too infrequently to be worth
2790 // the implementation effort, but if that ever changes, we could convert
2791 // such a pattern here.
2792 case PPC::CMPWI:
2793 case PPC::CMPLWI:
2794 case PPC::CMPDI:
2795 case PPC::CMPLDI: {
2796 // Doing this post-RA would require dataflow analysis to reliably find uses
2797 // of the CR register set by the compare.
2798 // No need to fixup killed/dead flag since this transformation is only valid
2799 // before RA.
2800 if (PostRA)
2801 return false;
2802 // If a compare-immediate is fed by an immediate and is itself an input of
2803 // an ISEL (the most common case) into a COPY of the correct register.
2804 bool Changed = false;
2805 Register DefReg = MI.getOperand(0).getReg();
2806 int64_t Comparand = MI.getOperand(2).getImm();
2807 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0 ?
2808 (Comparand | 0xFFFFFFFFFFFF0000) : Comparand;
2809
2810 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
2811 unsigned UseOpc = CompareUseMI.getOpcode();
2812 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
2813 continue;
2814 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
2815 Register TrueReg = CompareUseMI.getOperand(1).getReg();
2816 Register FalseReg = CompareUseMI.getOperand(2).getReg();
2817 unsigned RegToCopy = selectReg(SExtImm, SExtComparand, Opc, TrueReg,
2818 FalseReg, CRSubReg);
2819 if (RegToCopy == PPC::NoRegister)
2820 continue;
2821 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
2822 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
2823 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
2824 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
2825 CompareUseMI.RemoveOperand(3);
2826 CompareUseMI.RemoveOperand(2);
2827 continue;
2828 }
2829 LLVM_DEBUG(
2830 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
2831 LLVM_DEBUG(DefMI->dump(); MI.dump(); CompareUseMI.dump());
2832 LLVM_DEBUG(dbgs() << "Is converted to:\n");
2833 // Convert to copy and remove unneeded operands.
2834 CompareUseMI.setDesc(get(PPC::COPY));
2835 CompareUseMI.RemoveOperand(3);
2836 CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1);
2837 CmpIselsConverted++;
2838 Changed = true;
2839 LLVM_DEBUG(CompareUseMI.dump());
2840 }
2841 if (Changed)
2842 return true;
2843 // This may end up incremented multiple times since this function is called
2844 // during a fixed-point transformation, but it is only meant to indicate the
2845 // presence of this opportunity.
2846 MissedConvertibleImmediateInstrs++;
2847 return false;
2848 }
2849
2850 // Immediate forms - may simply be convertable to an LI.
2851 case PPC::ADDI:
2852 case PPC::ADDI8: {
2853 // Does the sum fit in a 16-bit signed field?
2854 int64_t Addend = MI.getOperand(2).getImm();
2855 if (isInt<16>(Addend + SExtImm)) {
2856 ReplaceWithLI = true;
2857 Is64BitLI = Opc == PPC::ADDI8;
2858 NewImm = Addend + SExtImm;
2859 break;
2860 }
2861 return false;
2862 }
2863 case PPC::RLDICL:
2864 case PPC::RLDICL_rec:
2865 case PPC::RLDICL_32:
2866 case PPC::RLDICL_32_64: {
2867 // Use APInt's rotate function.
2868 int64_t SH = MI.getOperand(2).getImm();
2869 int64_t MB = MI.getOperand(3).getImm();
2870 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
2871 SExtImm, true);
2872 InVal = InVal.rotl(SH);
2873 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2874 InVal &= Mask;
2875 // Can't replace negative values with an LI as that will sign-extend
2876 // and not clear the left bits. If we're setting the CR bit, we will use
2877 // ANDI_rec which won't sign extend, so that's safe.
2878 if (isUInt<15>(InVal.getSExtValue()) ||
2879 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
2880 ReplaceWithLI = true;
2881 Is64BitLI = Opc != PPC::RLDICL_32;
2882 NewImm = InVal.getSExtValue();
2883 SetCR = Opc == PPC::RLDICL_rec;
2884 break;
2885 }
2886 return false;
2887 }
2888 case PPC::RLWINM:
2889 case PPC::RLWINM8:
2890 case PPC::RLWINM_rec:
2891 case PPC::RLWINM8_rec: {
2892 int64_t SH = MI.getOperand(2).getImm();
2893 int64_t MB = MI.getOperand(3).getImm();
2894 int64_t ME = MI.getOperand(4).getImm();
2895 APInt InVal(32, SExtImm, true);
2896 InVal = InVal.rotl(SH);
2897 // Set the bits ( MB + 32 ) to ( ME + 32 ).
2898 uint64_t Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2899 InVal &= Mask;
2900 // Can't replace negative values with an LI as that will sign-extend
2901 // and not clear the left bits. If we're setting the CR bit, we will use
2902 // ANDI_rec which won't sign extend, so that's safe.
2903 bool ValueFits = isUInt<15>(InVal.getSExtValue());
2904 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
2905 isUInt<16>(InVal.getSExtValue()));
2906 if (ValueFits) {
2907 ReplaceWithLI = true;
2908 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
2909 NewImm = InVal.getSExtValue();
2910 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
2911 break;
2912 }
2913 return false;
2914 }
2915 case PPC::ORI:
2916 case PPC::ORI8:
2917 case PPC::XORI:
2918 case PPC::XORI8: {
2919 int64_t LogicalImm = MI.getOperand(2).getImm();
2920 int64_t Result = 0;
2921 if (Opc == PPC::ORI || Opc == PPC::ORI8)
2922 Result = LogicalImm | SExtImm;
2923 else
2924 Result = LogicalImm ^ SExtImm;
2925 if (isInt<16>(Result)) {
2926 ReplaceWithLI = true;
2927 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
2928 NewImm = Result;
2929 break;
2930 }
2931 return false;
2932 }
2933 }
2934
2935 if (ReplaceWithLI) {
2936 // We need to be careful with CR-setting instructions we're replacing.
2937 if (SetCR) {
2938 // We don't know anything about uses when we're out of SSA, so only
2939 // replace if the new immediate will be reproduced.
2940 bool ImmChanged = (SExtImm & NewImm) != NewImm;
2941 if (PostRA && ImmChanged)
2942 return false;
2943
2944 if (!PostRA) {
2945 // If the defining load-immediate has no other uses, we can just replace
2946 // the immediate with the new immediate.
2947 if (MRI->hasOneUse(DefMI->getOperand(0).getReg()))
2948 DefMI->getOperand(1).setImm(NewImm);
2949
2950 // If we're not using the GPR result of the CR-setting instruction, we
2951 // just need to and with zero/non-zero depending on the new immediate.
2952 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
2953 if (NewImm) {
2954 assert(Immediate && "Transformation converted zero to non-zero?");
2955 NewImm = Immediate;
2956 }
2957 }
2958 else if (ImmChanged)
2959 return false;
2960 }
2961 }
2962
2963 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
2964 LLVM_DEBUG(MI.dump());
2965 LLVM_DEBUG(dbgs() << "Fed by:\n");
2966 LLVM_DEBUG(DefMI->dump());
2967 LoadImmediateInfo LII;
2968 LII.Imm = NewImm;
2969 LII.Is64Bit = Is64BitLI;
2970 LII.SetCR = SetCR;
2971 // If we're setting the CR, the original load-immediate must be kept (as an
2972 // operand to ANDI_rec/ANDI8_rec).
2973 if (KilledDef && SetCR)
2974 *KilledDef = nullptr;
2975 replaceInstrWithLI(MI, LII);
2976
2977 // Fixup killed/dead flag after transformation.
2978 // Pattern:
2979 // ForwardingOperandReg = LI imm1
2980 // y = op2 imm2, ForwardingOperandReg(killed)
2981 if (IsForwardingOperandKilled)
2982 fixupIsDeadOrKill(*DefMI, MI, ForwardingOperandReg);
2983
2984 LLVM_DEBUG(dbgs() << "With:\n");
2985 LLVM_DEBUG(MI.dump());
2986 return true;
2987 }
2988 return false;
2989 }
2990
instrHasImmForm(unsigned Opc,bool IsVFReg,ImmInstrInfo & III,bool PostRA) const2991 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
2992 ImmInstrInfo &III, bool PostRA) const {
2993 // The vast majority of the instructions would need their operand 2 replaced
2994 // with an immediate when switching to the reg+imm form. A marked exception
2995 // are the update form loads/stores for which a constant operand 2 would need
2996 // to turn into a displacement and move operand 1 to the operand 2 position.
2997 III.ImmOpNo = 2;
2998 III.OpNoForForwarding = 2;
2999 III.ImmWidth = 16;
3000 III.ImmMustBeMultipleOf = 1;
3001 III.TruncateImmTo = 0;
3002 III.IsSummingOperands = false;
3003 switch (Opc) {
3004 default: return false;
3005 case PPC::ADD4:
3006 case PPC::ADD8:
3007 III.SignedImm = true;
3008 III.ZeroIsSpecialOrig = 0;
3009 III.ZeroIsSpecialNew = 1;
3010 III.IsCommutative = true;
3011 III.IsSummingOperands = true;
3012 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3013 break;
3014 case PPC::ADDC:
3015 case PPC::ADDC8:
3016 III.SignedImm = true;
3017 III.ZeroIsSpecialOrig = 0;
3018 III.ZeroIsSpecialNew = 0;
3019 III.IsCommutative = true;
3020 III.IsSummingOperands = true;
3021 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3022 break;
3023 case PPC::ADDC_rec:
3024 III.SignedImm = true;
3025 III.ZeroIsSpecialOrig = 0;
3026 III.ZeroIsSpecialNew = 0;
3027 III.IsCommutative = true;
3028 III.IsSummingOperands = true;
3029 III.ImmOpcode = PPC::ADDIC_rec;
3030 break;
3031 case PPC::SUBFC:
3032 case PPC::SUBFC8:
3033 III.SignedImm = true;
3034 III.ZeroIsSpecialOrig = 0;
3035 III.ZeroIsSpecialNew = 0;
3036 III.IsCommutative = false;
3037 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3038 break;
3039 case PPC::CMPW:
3040 case PPC::CMPD:
3041 III.SignedImm = true;
3042 III.ZeroIsSpecialOrig = 0;
3043 III.ZeroIsSpecialNew = 0;
3044 III.IsCommutative = false;
3045 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3046 break;
3047 case PPC::CMPLW:
3048 case PPC::CMPLD:
3049 III.SignedImm = false;
3050 III.ZeroIsSpecialOrig = 0;
3051 III.ZeroIsSpecialNew = 0;
3052 III.IsCommutative = false;
3053 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3054 break;
3055 case PPC::AND_rec:
3056 case PPC::AND8_rec:
3057 case PPC::OR:
3058 case PPC::OR8:
3059 case PPC::XOR:
3060 case PPC::XOR8:
3061 III.SignedImm = false;
3062 III.ZeroIsSpecialOrig = 0;
3063 III.ZeroIsSpecialNew = 0;
3064 III.IsCommutative = true;
3065 switch(Opc) {
3066 default: llvm_unreachable("Unknown opcode");
3067 case PPC::AND_rec:
3068 III.ImmOpcode = PPC::ANDI_rec;
3069 break;
3070 case PPC::AND8_rec:
3071 III.ImmOpcode = PPC::ANDI8_rec;
3072 break;
3073 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3074 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3075 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3076 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3077 }
3078 break;
3079 case PPC::RLWNM:
3080 case PPC::RLWNM8:
3081 case PPC::RLWNM_rec:
3082 case PPC::RLWNM8_rec:
3083 case PPC::SLW:
3084 case PPC::SLW8:
3085 case PPC::SLW_rec:
3086 case PPC::SLW8_rec:
3087 case PPC::SRW:
3088 case PPC::SRW8:
3089 case PPC::SRW_rec:
3090 case PPC::SRW8_rec:
3091 case PPC::SRAW:
3092 case PPC::SRAW_rec:
3093 III.SignedImm = false;
3094 III.ZeroIsSpecialOrig = 0;
3095 III.ZeroIsSpecialNew = 0;
3096 III.IsCommutative = false;
3097 // This isn't actually true, but the instructions ignore any of the
3098 // upper bits, so any immediate loaded with an LI is acceptable.
3099 // This does not apply to shift right algebraic because a value
3100 // out of range will produce a -1/0.
3101 III.ImmWidth = 16;
3102 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3103 Opc == PPC::RLWNM8_rec)
3104 III.TruncateImmTo = 5;
3105 else
3106 III.TruncateImmTo = 6;
3107 switch(Opc) {
3108 default: llvm_unreachable("Unknown opcode");
3109 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3110 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3111 case PPC::RLWNM_rec:
3112 III.ImmOpcode = PPC::RLWINM_rec;
3113 break;
3114 case PPC::RLWNM8_rec:
3115 III.ImmOpcode = PPC::RLWINM8_rec;
3116 break;
3117 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
3118 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
3119 case PPC::SLW_rec:
3120 III.ImmOpcode = PPC::RLWINM_rec;
3121 break;
3122 case PPC::SLW8_rec:
3123 III.ImmOpcode = PPC::RLWINM8_rec;
3124 break;
3125 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
3126 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
3127 case PPC::SRW_rec:
3128 III.ImmOpcode = PPC::RLWINM_rec;
3129 break;
3130 case PPC::SRW8_rec:
3131 III.ImmOpcode = PPC::RLWINM8_rec;
3132 break;
3133 case PPC::SRAW:
3134 III.ImmWidth = 5;
3135 III.TruncateImmTo = 0;
3136 III.ImmOpcode = PPC::SRAWI;
3137 break;
3138 case PPC::SRAW_rec:
3139 III.ImmWidth = 5;
3140 III.TruncateImmTo = 0;
3141 III.ImmOpcode = PPC::SRAWI_rec;
3142 break;
3143 }
3144 break;
3145 case PPC::RLDCL:
3146 case PPC::RLDCL_rec:
3147 case PPC::RLDCR:
3148 case PPC::RLDCR_rec:
3149 case PPC::SLD:
3150 case PPC::SLD_rec:
3151 case PPC::SRD:
3152 case PPC::SRD_rec:
3153 case PPC::SRAD:
3154 case PPC::SRAD_rec:
3155 III.SignedImm = false;
3156 III.ZeroIsSpecialOrig = 0;
3157 III.ZeroIsSpecialNew = 0;
3158 III.IsCommutative = false;
3159 // This isn't actually true, but the instructions ignore any of the
3160 // upper bits, so any immediate loaded with an LI is acceptable.
3161 // This does not apply to shift right algebraic because a value
3162 // out of range will produce a -1/0.
3163 III.ImmWidth = 16;
3164 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
3165 Opc == PPC::RLDCR_rec)
3166 III.TruncateImmTo = 6;
3167 else
3168 III.TruncateImmTo = 7;
3169 switch(Opc) {
3170 default: llvm_unreachable("Unknown opcode");
3171 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
3172 case PPC::RLDCL_rec:
3173 III.ImmOpcode = PPC::RLDICL_rec;
3174 break;
3175 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
3176 case PPC::RLDCR_rec:
3177 III.ImmOpcode = PPC::RLDICR_rec;
3178 break;
3179 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
3180 case PPC::SLD_rec:
3181 III.ImmOpcode = PPC::RLDICR_rec;
3182 break;
3183 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
3184 case PPC::SRD_rec:
3185 III.ImmOpcode = PPC::RLDICL_rec;
3186 break;
3187 case PPC::SRAD:
3188 III.ImmWidth = 6;
3189 III.TruncateImmTo = 0;
3190 III.ImmOpcode = PPC::SRADI;
3191 break;
3192 case PPC::SRAD_rec:
3193 III.ImmWidth = 6;
3194 III.TruncateImmTo = 0;
3195 III.ImmOpcode = PPC::SRADI_rec;
3196 break;
3197 }
3198 break;
3199 // Loads and stores:
3200 case PPC::LBZX:
3201 case PPC::LBZX8:
3202 case PPC::LHZX:
3203 case PPC::LHZX8:
3204 case PPC::LHAX:
3205 case PPC::LHAX8:
3206 case PPC::LWZX:
3207 case PPC::LWZX8:
3208 case PPC::LWAX:
3209 case PPC::LDX:
3210 case PPC::LFSX:
3211 case PPC::LFDX:
3212 case PPC::STBX:
3213 case PPC::STBX8:
3214 case PPC::STHX:
3215 case PPC::STHX8:
3216 case PPC::STWX:
3217 case PPC::STWX8:
3218 case PPC::STDX:
3219 case PPC::STFSX:
3220 case PPC::STFDX:
3221 III.SignedImm = true;
3222 III.ZeroIsSpecialOrig = 1;
3223 III.ZeroIsSpecialNew = 2;
3224 III.IsCommutative = true;
3225 III.IsSummingOperands = true;
3226 III.ImmOpNo = 1;
3227 III.OpNoForForwarding = 2;
3228 switch(Opc) {
3229 default: llvm_unreachable("Unknown opcode");
3230 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
3231 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
3232 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
3233 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
3234 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
3235 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
3236 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
3237 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
3238 case PPC::LWAX:
3239 III.ImmOpcode = PPC::LWA;
3240 III.ImmMustBeMultipleOf = 4;
3241 break;
3242 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
3243 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
3244 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
3245 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
3246 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
3247 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
3248 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
3249 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
3250 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
3251 case PPC::STDX:
3252 III.ImmOpcode = PPC::STD;
3253 III.ImmMustBeMultipleOf = 4;
3254 break;
3255 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
3256 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
3257 }
3258 break;
3259 case PPC::LBZUX:
3260 case PPC::LBZUX8:
3261 case PPC::LHZUX:
3262 case PPC::LHZUX8:
3263 case PPC::LHAUX:
3264 case PPC::LHAUX8:
3265 case PPC::LWZUX:
3266 case PPC::LWZUX8:
3267 case PPC::LDUX:
3268 case PPC::LFSUX:
3269 case PPC::LFDUX:
3270 case PPC::STBUX:
3271 case PPC::STBUX8:
3272 case PPC::STHUX:
3273 case PPC::STHUX8:
3274 case PPC::STWUX:
3275 case PPC::STWUX8:
3276 case PPC::STDUX:
3277 case PPC::STFSUX:
3278 case PPC::STFDUX:
3279 III.SignedImm = true;
3280 III.ZeroIsSpecialOrig = 2;
3281 III.ZeroIsSpecialNew = 3;
3282 III.IsCommutative = false;
3283 III.IsSummingOperands = true;
3284 III.ImmOpNo = 2;
3285 III.OpNoForForwarding = 3;
3286 switch(Opc) {
3287 default: llvm_unreachable("Unknown opcode");
3288 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
3289 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
3290 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
3291 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
3292 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
3293 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
3294 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
3295 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
3296 case PPC::LDUX:
3297 III.ImmOpcode = PPC::LDU;
3298 III.ImmMustBeMultipleOf = 4;
3299 break;
3300 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
3301 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
3302 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
3303 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
3304 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
3305 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
3306 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
3307 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
3308 case PPC::STDUX:
3309 III.ImmOpcode = PPC::STDU;
3310 III.ImmMustBeMultipleOf = 4;
3311 break;
3312 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
3313 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
3314 }
3315 break;
3316 // Power9 and up only. For some of these, the X-Form version has access to all
3317 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
3318 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
3319 // into or stored from is one of the VR registers.
3320 case PPC::LXVX:
3321 case PPC::LXSSPX:
3322 case PPC::LXSDX:
3323 case PPC::STXVX:
3324 case PPC::STXSSPX:
3325 case PPC::STXSDX:
3326 case PPC::XFLOADf32:
3327 case PPC::XFLOADf64:
3328 case PPC::XFSTOREf32:
3329 case PPC::XFSTOREf64:
3330 if (!Subtarget.hasP9Vector())
3331 return false;
3332 III.SignedImm = true;
3333 III.ZeroIsSpecialOrig = 1;
3334 III.ZeroIsSpecialNew = 2;
3335 III.IsCommutative = true;
3336 III.IsSummingOperands = true;
3337 III.ImmOpNo = 1;
3338 III.OpNoForForwarding = 2;
3339 III.ImmMustBeMultipleOf = 4;
3340 switch(Opc) {
3341 default: llvm_unreachable("Unknown opcode");
3342 case PPC::LXVX:
3343 III.ImmOpcode = PPC::LXV;
3344 III.ImmMustBeMultipleOf = 16;
3345 break;
3346 case PPC::LXSSPX:
3347 if (PostRA) {
3348 if (IsVFReg)
3349 III.ImmOpcode = PPC::LXSSP;
3350 else {
3351 III.ImmOpcode = PPC::LFS;
3352 III.ImmMustBeMultipleOf = 1;
3353 }
3354 break;
3355 }
3356 LLVM_FALLTHROUGH;
3357 case PPC::XFLOADf32:
3358 III.ImmOpcode = PPC::DFLOADf32;
3359 break;
3360 case PPC::LXSDX:
3361 if (PostRA) {
3362 if (IsVFReg)
3363 III.ImmOpcode = PPC::LXSD;
3364 else {
3365 III.ImmOpcode = PPC::LFD;
3366 III.ImmMustBeMultipleOf = 1;
3367 }
3368 break;
3369 }
3370 LLVM_FALLTHROUGH;
3371 case PPC::XFLOADf64:
3372 III.ImmOpcode = PPC::DFLOADf64;
3373 break;
3374 case PPC::STXVX:
3375 III.ImmOpcode = PPC::STXV;
3376 III.ImmMustBeMultipleOf = 16;
3377 break;
3378 case PPC::STXSSPX:
3379 if (PostRA) {
3380 if (IsVFReg)
3381 III.ImmOpcode = PPC::STXSSP;
3382 else {
3383 III.ImmOpcode = PPC::STFS;
3384 III.ImmMustBeMultipleOf = 1;
3385 }
3386 break;
3387 }
3388 LLVM_FALLTHROUGH;
3389 case PPC::XFSTOREf32:
3390 III.ImmOpcode = PPC::DFSTOREf32;
3391 break;
3392 case PPC::STXSDX:
3393 if (PostRA) {
3394 if (IsVFReg)
3395 III.ImmOpcode = PPC::STXSD;
3396 else {
3397 III.ImmOpcode = PPC::STFD;
3398 III.ImmMustBeMultipleOf = 1;
3399 }
3400 break;
3401 }
3402 LLVM_FALLTHROUGH;
3403 case PPC::XFSTOREf64:
3404 III.ImmOpcode = PPC::DFSTOREf64;
3405 break;
3406 }
3407 break;
3408 }
3409 return true;
3410 }
3411
3412 // Utility function for swaping two arbitrary operands of an instruction.
swapMIOperands(MachineInstr & MI,unsigned Op1,unsigned Op2)3413 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
3414 assert(Op1 != Op2 && "Cannot swap operand with itself.");
3415
3416 unsigned MaxOp = std::max(Op1, Op2);
3417 unsigned MinOp = std::min(Op1, Op2);
3418 MachineOperand MOp1 = MI.getOperand(MinOp);
3419 MachineOperand MOp2 = MI.getOperand(MaxOp);
3420 MI.RemoveOperand(std::max(Op1, Op2));
3421 MI.RemoveOperand(std::min(Op1, Op2));
3422
3423 // If the operands we are swapping are the two at the end (the common case)
3424 // we can just remove both and add them in the opposite order.
3425 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
3426 MI.addOperand(MOp2);
3427 MI.addOperand(MOp1);
3428 } else {
3429 // Store all operands in a temporary vector, remove them and re-add in the
3430 // right order.
3431 SmallVector<MachineOperand, 2> MOps;
3432 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
3433 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
3434 MOps.push_back(MI.getOperand(i));
3435 MI.RemoveOperand(i);
3436 }
3437 // MOp2 needs to be added next.
3438 MI.addOperand(MOp2);
3439 // Now add the rest.
3440 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
3441 if (i == MaxOp)
3442 MI.addOperand(MOp1);
3443 else {
3444 MI.addOperand(MOps.back());
3445 MOps.pop_back();
3446 }
3447 }
3448 }
3449 }
3450
3451 // Check if the 'MI' that has the index OpNoForForwarding
3452 // meets the requirement described in the ImmInstrInfo.
isUseMIElgibleForForwarding(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding) const3453 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
3454 const ImmInstrInfo &III,
3455 unsigned OpNoForForwarding
3456 ) const {
3457 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
3458 // would not work pre-RA, we can only do the check post RA.
3459 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3460 if (MRI.isSSA())
3461 return false;
3462
3463 // Cannot do the transform if MI isn't summing the operands.
3464 if (!III.IsSummingOperands)
3465 return false;
3466
3467 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
3468 if (!III.ZeroIsSpecialOrig)
3469 return false;
3470
3471 // We cannot do the transform if the operand we are trying to replace
3472 // isn't the same as the operand the instruction allows.
3473 if (OpNoForForwarding != III.OpNoForForwarding)
3474 return false;
3475
3476 // Check if the instruction we are trying to transform really has
3477 // the special zero register as its operand.
3478 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
3479 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
3480 return false;
3481
3482 // This machine instruction is convertible if it is,
3483 // 1. summing the operands.
3484 // 2. one of the operands is special zero register.
3485 // 3. the operand we are trying to replace is allowed by the MI.
3486 return true;
3487 }
3488
3489 // Check if the DefMI is the add inst and set the ImmMO and RegMO
3490 // accordingly.
isDefMIElgibleForForwarding(MachineInstr & DefMI,const ImmInstrInfo & III,MachineOperand * & ImmMO,MachineOperand * & RegMO) const3491 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
3492 const ImmInstrInfo &III,
3493 MachineOperand *&ImmMO,
3494 MachineOperand *&RegMO) const {
3495 unsigned Opc = DefMI.getOpcode();
3496 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
3497 return false;
3498
3499 assert(DefMI.getNumOperands() >= 3 &&
3500 "Add inst must have at least three operands");
3501 RegMO = &DefMI.getOperand(1);
3502 ImmMO = &DefMI.getOperand(2);
3503
3504 // This DefMI is elgible for forwarding if it is:
3505 // 1. add inst
3506 // 2. one of the operands is Imm/CPI/Global.
3507 return isAnImmediateOperand(*ImmMO);
3508 }
3509
isRegElgibleForForwarding(const MachineOperand & RegMO,const MachineInstr & DefMI,const MachineInstr & MI,bool KillDefMI,bool & IsFwdFeederRegKilled) const3510 bool PPCInstrInfo::isRegElgibleForForwarding(
3511 const MachineOperand &RegMO, const MachineInstr &DefMI,
3512 const MachineInstr &MI, bool KillDefMI,
3513 bool &IsFwdFeederRegKilled) const {
3514 // x = addi y, imm
3515 // ...
3516 // z = lfdx 0, x -> z = lfd imm(y)
3517 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
3518 // of "y" between the DEF of "x" and "z".
3519 // The query is only valid post RA.
3520 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3521 if (MRI.isSSA())
3522 return false;
3523
3524 Register Reg = RegMO.getReg();
3525
3526 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
3527 MachineBasicBlock::const_reverse_iterator It = MI;
3528 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
3529 It++;
3530 for (; It != E; ++It) {
3531 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3532 return false;
3533 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3534 IsFwdFeederRegKilled = true;
3535 // Made it to DefMI without encountering a clobber.
3536 if ((&*It) == &DefMI)
3537 break;
3538 }
3539 assert((&*It) == &DefMI && "DefMI is missing");
3540
3541 // If DefMI also defines the register to be forwarded, we can only forward it
3542 // if DefMI is being erased.
3543 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
3544 return KillDefMI;
3545
3546 return true;
3547 }
3548
isImmElgibleForForwarding(const MachineOperand & ImmMO,const MachineInstr & DefMI,const ImmInstrInfo & III,int64_t & Imm) const3549 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
3550 const MachineInstr &DefMI,
3551 const ImmInstrInfo &III,
3552 int64_t &Imm) const {
3553 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
3554 if (DefMI.getOpcode() == PPC::ADDItocL) {
3555 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
3556 // However, we know that, it is 16-bit width, and has the alignment of 4.
3557 // Check if the instruction met the requirement.
3558 if (III.ImmMustBeMultipleOf > 4 ||
3559 III.TruncateImmTo || III.ImmWidth != 16)
3560 return false;
3561
3562 // Going from XForm to DForm loads means that the displacement needs to be
3563 // not just an immediate but also a multiple of 4, or 16 depending on the
3564 // load. A DForm load cannot be represented if it is a multiple of say 2.
3565 // XForm loads do not have this restriction.
3566 if (ImmMO.isGlobal() &&
3567 ImmMO.getGlobal()->getAlignment() < III.ImmMustBeMultipleOf)
3568 return false;
3569
3570 return true;
3571 }
3572
3573 if (ImmMO.isImm()) {
3574 // It is Imm, we need to check if the Imm fit the range.
3575 int64_t Immediate = ImmMO.getImm();
3576 // Sign-extend to 64-bits.
3577 Imm = ((uint64_t)Immediate & ~0x7FFFuLL) != 0 ?
3578 (Immediate | 0xFFFFFFFFFFFF0000) : Immediate;
3579
3580 if (Imm % III.ImmMustBeMultipleOf)
3581 return false;
3582 if (III.TruncateImmTo)
3583 Imm &= ((1 << III.TruncateImmTo) - 1);
3584 if (III.SignedImm) {
3585 APInt ActualValue(64, Imm, true);
3586 if (!ActualValue.isSignedIntN(III.ImmWidth))
3587 return false;
3588 } else {
3589 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
3590 if ((uint64_t)Imm > UnsignedMax)
3591 return false;
3592 }
3593 }
3594 else
3595 return false;
3596
3597 // This ImmMO is forwarded if it meets the requriement describle
3598 // in ImmInstrInfo
3599 return true;
3600 }
3601
3602 // If an X-Form instruction is fed by an add-immediate and one of its operands
3603 // is the literal zero, attempt to forward the source of the add-immediate to
3604 // the corresponding D-Form instruction with the displacement coming from
3605 // the immediate being added.
transformToImmFormFedByAdd(MachineInstr & MI,const ImmInstrInfo & III,unsigned OpNoForForwarding,MachineInstr & DefMI,bool KillDefMI) const3606 bool PPCInstrInfo::transformToImmFormFedByAdd(
3607 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
3608 MachineInstr &DefMI, bool KillDefMI) const {
3609 // RegMO ImmMO
3610 // | |
3611 // x = addi reg, imm <----- DefMI
3612 // y = op 0 , x <----- MI
3613 // |
3614 // OpNoForForwarding
3615 // Check if the MI meet the requirement described in the III.
3616 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
3617 return false;
3618
3619 // Check if the DefMI meet the requirement
3620 // described in the III. If yes, set the ImmMO and RegMO accordingly.
3621 MachineOperand *ImmMO = nullptr;
3622 MachineOperand *RegMO = nullptr;
3623 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3624 return false;
3625 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3626
3627 // As we get the Imm operand now, we need to check if the ImmMO meet
3628 // the requirement described in the III. If yes set the Imm.
3629 int64_t Imm = 0;
3630 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
3631 return false;
3632
3633 bool IsFwdFeederRegKilled = false;
3634 // Check if the RegMO can be forwarded to MI.
3635 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
3636 IsFwdFeederRegKilled))
3637 return false;
3638
3639 // Get killed info in case fixup needed after transformation.
3640 unsigned ForwardKilledOperandReg = ~0U;
3641 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3642 bool PostRA = !MRI.isSSA();
3643 if (PostRA && MI.getOperand(OpNoForForwarding).isKill())
3644 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg();
3645
3646 // We know that, the MI and DefMI both meet the pattern, and
3647 // the Imm also meet the requirement with the new Imm-form.
3648 // It is safe to do the transformation now.
3649 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3650 LLVM_DEBUG(MI.dump());
3651 LLVM_DEBUG(dbgs() << "Fed by:\n");
3652 LLVM_DEBUG(DefMI.dump());
3653
3654 // Update the base reg first.
3655 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
3656 false, false,
3657 RegMO->isKill());
3658
3659 // Then, update the imm.
3660 if (ImmMO->isImm()) {
3661 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
3662 // directly.
3663 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
3664 }
3665 else {
3666 // Otherwise, it is Constant Pool Index(CPI) or Global,
3667 // which is relocation in fact. We need to replace the special zero
3668 // register with ImmMO.
3669 // Before that, we need to fixup the target flags for imm.
3670 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
3671 if (DefMI.getOpcode() == PPC::ADDItocL)
3672 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
3673
3674 // MI didn't have the interface such as MI.setOperand(i) though
3675 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
3676 // ImmMO, we need to remove ZERO operand and all the operands behind it,
3677 // and, add the ImmMO, then, move back all the operands behind ZERO.
3678 SmallVector<MachineOperand, 2> MOps;
3679 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
3680 MOps.push_back(MI.getOperand(i));
3681 MI.RemoveOperand(i);
3682 }
3683
3684 // Remove the last MO in the list, which is ZERO operand in fact.
3685 MOps.pop_back();
3686 // Add the imm operand.
3687 MI.addOperand(*ImmMO);
3688 // Now add the rest back.
3689 for (auto &MO : MOps)
3690 MI.addOperand(MO);
3691 }
3692
3693 // Update the opcode.
3694 MI.setDesc(get(III.ImmOpcode));
3695
3696 // Fix up killed/dead flag after transformation.
3697 // Pattern 1:
3698 // x = ADD KilledFwdFeederReg, imm
3699 // n = opn KilledFwdFeederReg(killed), regn
3700 // y = XOP 0, x
3701 // Pattern 2:
3702 // x = ADD reg(killed), imm
3703 // y = XOP 0, x
3704 if (IsFwdFeederRegKilled || RegMO->isKill())
3705 fixupIsDeadOrKill(DefMI, MI, RegMO->getReg());
3706 // Pattern 3:
3707 // ForwardKilledOperandReg = ADD reg, imm
3708 // y = XOP 0, ForwardKilledOperandReg(killed)
3709 if (ForwardKilledOperandReg != ~0U)
3710 fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
3711
3712 LLVM_DEBUG(dbgs() << "With:\n");
3713 LLVM_DEBUG(MI.dump());
3714
3715 return true;
3716 }
3717
transformToImmFormFedByLI(MachineInstr & MI,const ImmInstrInfo & III,unsigned ConstantOpNo,MachineInstr & DefMI,int64_t Imm) const3718 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
3719 const ImmInstrInfo &III,
3720 unsigned ConstantOpNo,
3721 MachineInstr &DefMI,
3722 int64_t Imm) const {
3723 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3724 bool PostRA = !MRI.isSSA();
3725 // Exit early if we can't convert this.
3726 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
3727 return false;
3728 if (Imm % III.ImmMustBeMultipleOf)
3729 return false;
3730 if (III.TruncateImmTo)
3731 Imm &= ((1 << III.TruncateImmTo) - 1);
3732 if (III.SignedImm) {
3733 APInt ActualValue(64, Imm, true);
3734 if (!ActualValue.isSignedIntN(III.ImmWidth))
3735 return false;
3736 } else {
3737 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
3738 if ((uint64_t)Imm > UnsignedMax)
3739 return false;
3740 }
3741
3742 // If we're post-RA, the instructions don't agree on whether register zero is
3743 // special, we can transform this as long as the register operand that will
3744 // end up in the location where zero is special isn't R0.
3745 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
3746 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
3747 III.ZeroIsSpecialNew + 1;
3748 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
3749 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
3750 // If R0 is in the operand where zero is special for the new instruction,
3751 // it is unsafe to transform if the constant operand isn't that operand.
3752 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
3753 ConstantOpNo != III.ZeroIsSpecialNew)
3754 return false;
3755 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
3756 ConstantOpNo != PosForOrigZero)
3757 return false;
3758 }
3759
3760 // Get killed info in case fixup needed after transformation.
3761 unsigned ForwardKilledOperandReg = ~0U;
3762 if (PostRA && MI.getOperand(ConstantOpNo).isKill())
3763 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg();
3764
3765 unsigned Opc = MI.getOpcode();
3766 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
3767 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
3768 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
3769 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
3770 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
3771 Opc == PPC::SRD || Opc == PPC::SRD_rec;
3772 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
3773 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
3774 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
3775 Opc == PPC::SRD_rec;
3776
3777 MI.setDesc(get(III.ImmOpcode));
3778 if (ConstantOpNo == III.OpNoForForwarding) {
3779 // Converting shifts to immediate form is a bit tricky since they may do
3780 // one of three things:
3781 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
3782 // 2. If the shift amount is zero, the result is unchanged (save for maybe
3783 // setting CR0)
3784 // 3. If the shift amount is in [1, OpSize), it's just a shift
3785 if (SpecialShift32 || SpecialShift64) {
3786 LoadImmediateInfo LII;
3787 LII.Imm = 0;
3788 LII.SetCR = SetCR;
3789 LII.Is64Bit = SpecialShift64;
3790 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
3791 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
3792 replaceInstrWithLI(MI, LII);
3793 // Shifts by zero don't change the value. If we don't need to set CR0,
3794 // just convert this to a COPY. Can't do this post-RA since we've already
3795 // cleaned up the copies.
3796 else if (!SetCR && ShAmt == 0 && !PostRA) {
3797 MI.RemoveOperand(2);
3798 MI.setDesc(get(PPC::COPY));
3799 } else {
3800 // The 32 bit and 64 bit instructions are quite different.
3801 if (SpecialShift32) {
3802 // Left shifts use (N, 0, 31-N).
3803 // Right shifts use (32-N, N, 31) if 0 < N < 32.
3804 // use (0, 0, 31) if N == 0.
3805 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
3806 uint64_t MB = RightShift ? ShAmt : 0;
3807 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
3808 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
3809 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
3810 .addImm(ME);
3811 } else {
3812 // Left shifts use (N, 63-N).
3813 // Right shifts use (64-N, N) if 0 < N < 64.
3814 // use (0, 0) if N == 0.
3815 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
3816 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
3817 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
3818 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
3819 }
3820 }
3821 } else
3822 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
3823 }
3824 // Convert commutative instructions (switch the operands and convert the
3825 // desired one to an immediate.
3826 else if (III.IsCommutative) {
3827 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
3828 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
3829 } else
3830 llvm_unreachable("Should have exited early!");
3831
3832 // For instructions for which the constant register replaces a different
3833 // operand than where the immediate goes, we need to swap them.
3834 if (III.OpNoForForwarding != III.ImmOpNo)
3835 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
3836
3837 // If the special R0/X0 register index are different for original instruction
3838 // and new instruction, we need to fix up the register class in new
3839 // instruction.
3840 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
3841 if (III.ZeroIsSpecialNew) {
3842 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
3843 // need to fix up register class.
3844 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
3845 if (Register::isVirtualRegister(RegToModify)) {
3846 const TargetRegisterClass *NewRC =
3847 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
3848 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
3849 MRI.setRegClass(RegToModify, NewRC);
3850 }
3851 }
3852 }
3853
3854 // Fix up killed/dead flag after transformation.
3855 // Pattern:
3856 // ForwardKilledOperandReg = LI imm
3857 // y = XOP reg, ForwardKilledOperandReg(killed)
3858 if (ForwardKilledOperandReg != ~0U)
3859 fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
3860 return true;
3861 }
3862
3863 const TargetRegisterClass *
updatedRC(const TargetRegisterClass * RC) const3864 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
3865 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
3866 return &PPC::VSRCRegClass;
3867 return RC;
3868 }
3869
getRecordFormOpcode(unsigned Opcode)3870 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
3871 return PPC::getRecordFormOpcode(Opcode);
3872 }
3873
3874 // This function returns true if the machine instruction
3875 // always outputs a value by sign-extending a 32 bit value,
3876 // i.e. 0 to 31-th bits are same as 32-th bit.
isSignExtendingOp(const MachineInstr & MI)3877 static bool isSignExtendingOp(const MachineInstr &MI) {
3878 int Opcode = MI.getOpcode();
3879 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS ||
3880 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec ||
3881 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA ||
3882 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 ||
3883 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 ||
3884 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX ||
3885 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU ||
3886 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
3887 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 ||
3888 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX ||
3889 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB ||
3890 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH ||
3891 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 ||
3892 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW ||
3893 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 ||
3894 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 ||
3895 Opcode == PPC::EXTSB8_32_64)
3896 return true;
3897
3898 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33)
3899 return true;
3900
3901 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
3902 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
3903 MI.getOperand(3).getImm() > 0 &&
3904 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
3905 return true;
3906
3907 return false;
3908 }
3909
3910 // This function returns true if the machine instruction
3911 // always outputs zeros in higher 32 bits.
isZeroExtendingOp(const MachineInstr & MI)3912 static bool isZeroExtendingOp(const MachineInstr &MI) {
3913 int Opcode = MI.getOpcode();
3914 // The 16-bit immediate is sign-extended in li/lis.
3915 // If the most significant bit is zero, all higher bits are zero.
3916 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
3917 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
3918 int64_t Imm = MI.getOperand(1).getImm();
3919 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
3920 return true;
3921 }
3922
3923 // We have some variations of rotate-and-mask instructions
3924 // that clear higher 32-bits.
3925 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
3926 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
3927 Opcode == PPC::RLDICL_32_64) &&
3928 MI.getOperand(3).getImm() >= 32)
3929 return true;
3930
3931 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
3932 MI.getOperand(3).getImm() >= 32 &&
3933 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm())
3934 return true;
3935
3936 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
3937 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
3938 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
3939 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
3940 return true;
3941
3942 // There are other instructions that clear higher 32-bits.
3943 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec ||
3944 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec ||
3945 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 ||
3946 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec ||
3947 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec ||
3948 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW ||
3949 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec ||
3950 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI ||
3951 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI ||
3952 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX ||
3953 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX ||
3954 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX ||
3955 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ ||
3956 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX ||
3957 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 ||
3958 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 ||
3959 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 ||
3960 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 ||
3961 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
3962 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec ||
3963 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec ||
3964 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec ||
3965 Opcode == PPC::MFVSRWZ)
3966 return true;
3967
3968 return false;
3969 }
3970
3971 // This function returns true if the input MachineInstr is a TOC save
3972 // instruction.
isTOCSaveMI(const MachineInstr & MI) const3973 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
3974 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
3975 return false;
3976 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
3977 unsigned StackOffset = MI.getOperand(1).getImm();
3978 Register StackReg = MI.getOperand(2).getReg();
3979 if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset)
3980 return true;
3981
3982 return false;
3983 }
3984
3985 // We limit the max depth to track incoming values of PHIs or binary ops
3986 // (e.g. AND) to avoid excessive cost.
3987 const unsigned MAX_DEPTH = 1;
3988
3989 bool
isSignOrZeroExtended(const MachineInstr & MI,bool SignExt,const unsigned Depth) const3990 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
3991 const unsigned Depth) const {
3992 const MachineFunction *MF = MI.getParent()->getParent();
3993 const MachineRegisterInfo *MRI = &MF->getRegInfo();
3994
3995 // If we know this instruction returns sign- or zero-extended result,
3996 // return true.
3997 if (SignExt ? isSignExtendingOp(MI):
3998 isZeroExtendingOp(MI))
3999 return true;
4000
4001 switch (MI.getOpcode()) {
4002 case PPC::COPY: {
4003 Register SrcReg = MI.getOperand(1).getReg();
4004
4005 // In both ELFv1 and v2 ABI, method parameters and the return value
4006 // are sign- or zero-extended.
4007 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
4008 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
4009 // We check the ZExt/SExt flags for a method parameter.
4010 if (MI.getParent()->getBasicBlock() ==
4011 &MF->getFunction().getEntryBlock()) {
4012 Register VReg = MI.getOperand(0).getReg();
4013 if (MF->getRegInfo().isLiveIn(VReg))
4014 return SignExt ? FuncInfo->isLiveInSExt(VReg) :
4015 FuncInfo->isLiveInZExt(VReg);
4016 }
4017
4018 // For a method return value, we check the ZExt/SExt flags in attribute.
4019 // We assume the following code sequence for method call.
4020 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
4021 // BL8_NOP @func,...
4022 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
4023 // %5 = COPY %x3; G8RC:%5
4024 if (SrcReg == PPC::X3) {
4025 const MachineBasicBlock *MBB = MI.getParent();
4026 MachineBasicBlock::const_instr_iterator II =
4027 MachineBasicBlock::const_instr_iterator(&MI);
4028 if (II != MBB->instr_begin() &&
4029 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) {
4030 const MachineInstr &CallMI = *(--II);
4031 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) {
4032 const Function *CalleeFn =
4033 dyn_cast<Function>(CallMI.getOperand(0).getGlobal());
4034 if (!CalleeFn)
4035 return false;
4036 const IntegerType *IntTy =
4037 dyn_cast<IntegerType>(CalleeFn->getReturnType());
4038 const AttributeSet &Attrs =
4039 CalleeFn->getAttributes().getRetAttributes();
4040 if (IntTy && IntTy->getBitWidth() <= 32)
4041 return Attrs.hasAttribute(SignExt ? Attribute::SExt :
4042 Attribute::ZExt);
4043 }
4044 }
4045 }
4046 }
4047
4048 // If this is a copy from another register, we recursively check source.
4049 if (!Register::isVirtualRegister(SrcReg))
4050 return false;
4051 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4052 if (SrcMI != NULL)
4053 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4054
4055 return false;
4056 }
4057
4058 case PPC::ANDI_rec:
4059 case PPC::ANDIS_rec:
4060 case PPC::ORI:
4061 case PPC::ORIS:
4062 case PPC::XORI:
4063 case PPC::XORIS:
4064 case PPC::ANDI8_rec:
4065 case PPC::ANDIS8_rec:
4066 case PPC::ORI8:
4067 case PPC::ORIS8:
4068 case PPC::XORI8:
4069 case PPC::XORIS8: {
4070 // logical operation with 16-bit immediate does not change the upper bits.
4071 // So, we track the operand register as we do for register copy.
4072 Register SrcReg = MI.getOperand(1).getReg();
4073 if (!Register::isVirtualRegister(SrcReg))
4074 return false;
4075 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4076 if (SrcMI != NULL)
4077 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4078
4079 return false;
4080 }
4081
4082 // If all incoming values are sign-/zero-extended,
4083 // the output of OR, ISEL or PHI is also sign-/zero-extended.
4084 case PPC::OR:
4085 case PPC::OR8:
4086 case PPC::ISEL:
4087 case PPC::PHI: {
4088 if (Depth >= MAX_DEPTH)
4089 return false;
4090
4091 // The input registers for PHI are operand 1, 3, ...
4092 // The input registers for others are operand 1 and 2.
4093 unsigned E = 3, D = 1;
4094 if (MI.getOpcode() == PPC::PHI) {
4095 E = MI.getNumOperands();
4096 D = 2;
4097 }
4098
4099 for (unsigned I = 1; I != E; I += D) {
4100 if (MI.getOperand(I).isReg()) {
4101 Register SrcReg = MI.getOperand(I).getReg();
4102 if (!Register::isVirtualRegister(SrcReg))
4103 return false;
4104 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4105 if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1))
4106 return false;
4107 }
4108 else
4109 return false;
4110 }
4111 return true;
4112 }
4113
4114 // If at least one of the incoming values of an AND is zero extended
4115 // then the output is also zero-extended. If both of the incoming values
4116 // are sign-extended then the output is also sign extended.
4117 case PPC::AND:
4118 case PPC::AND8: {
4119 if (Depth >= MAX_DEPTH)
4120 return false;
4121
4122 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg());
4123
4124 Register SrcReg1 = MI.getOperand(1).getReg();
4125 Register SrcReg2 = MI.getOperand(2).getReg();
4126
4127 if (!Register::isVirtualRegister(SrcReg1) ||
4128 !Register::isVirtualRegister(SrcReg2))
4129 return false;
4130
4131 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1);
4132 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2);
4133 if (!MISrc1 || !MISrc2)
4134 return false;
4135
4136 if(SignExt)
4137 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) &&
4138 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4139 else
4140 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) ||
4141 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4142 }
4143
4144 default:
4145 break;
4146 }
4147 return false;
4148 }
4149
isBDNZ(unsigned Opcode) const4150 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
4151 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
4152 }
4153
4154 namespace {
4155 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
4156 MachineInstr *Loop, *EndLoop, *LoopCount;
4157 MachineFunction *MF;
4158 const TargetInstrInfo *TII;
4159 int64_t TripCount;
4160
4161 public:
PPCPipelinerLoopInfo(MachineInstr * Loop,MachineInstr * EndLoop,MachineInstr * LoopCount)4162 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
4163 MachineInstr *LoopCount)
4164 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
4165 MF(Loop->getParent()->getParent()),
4166 TII(MF->getSubtarget().getInstrInfo()) {
4167 // Inspect the Loop instruction up-front, as it may be deleted when we call
4168 // createTripCountGreaterCondition.
4169 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
4170 TripCount = LoopCount->getOperand(1).getImm();
4171 else
4172 TripCount = -1;
4173 }
4174
shouldIgnoreForPipelining(const MachineInstr * MI) const4175 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4176 // Only ignore the terminator.
4177 return MI == EndLoop;
4178 }
4179
4180 Optional<bool>
createTripCountGreaterCondition(int TC,MachineBasicBlock & MBB,SmallVectorImpl<MachineOperand> & Cond)4181 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
4182 SmallVectorImpl<MachineOperand> &Cond) override {
4183 if (TripCount == -1) {
4184 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4185 // so we don't need to generate any thing here.
4186 Cond.push_back(MachineOperand::CreateImm(0));
4187 Cond.push_back(MachineOperand::CreateReg(
4188 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
4189 true));
4190 return {};
4191 }
4192
4193 return TripCount > TC;
4194 }
4195
setPreheader(MachineBasicBlock * NewPreheader)4196 void setPreheader(MachineBasicBlock *NewPreheader) override {
4197 // Do nothing. We want the LOOP setup instruction to stay in the *old*
4198 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
4199 }
4200
adjustTripCount(int TripCountAdjust)4201 void adjustTripCount(int TripCountAdjust) override {
4202 // If the loop trip count is a compile-time value, then just change the
4203 // value.
4204 if (LoopCount->getOpcode() == PPC::LI8 ||
4205 LoopCount->getOpcode() == PPC::LI) {
4206 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
4207 LoopCount->getOperand(1).setImm(TripCount);
4208 return;
4209 }
4210
4211 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4212 // so we don't need to generate any thing here.
4213 }
4214
disposed()4215 void disposed() override {
4216 Loop->eraseFromParent();
4217 // Ensure the loop setup instruction is deleted too.
4218 LoopCount->eraseFromParent();
4219 }
4220 };
4221 } // namespace
4222
4223 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
analyzeLoopForPipelining(MachineBasicBlock * LoopBB) const4224 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
4225 // We really "analyze" only hardware loops right now.
4226 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
4227 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
4228 if (Preheader == LoopBB)
4229 Preheader = *std::next(LoopBB->pred_begin());
4230 MachineFunction *MF = Preheader->getParent();
4231
4232 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
4233 SmallPtrSet<MachineBasicBlock *, 8> Visited;
4234 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
4235 Register LoopCountReg = LoopInst->getOperand(0).getReg();
4236 MachineRegisterInfo &MRI = MF->getRegInfo();
4237 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
4238 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
4239 }
4240 }
4241 return nullptr;
4242 }
4243
findLoopInstr(MachineBasicBlock & PreHeader,SmallPtrSet<MachineBasicBlock *,8> & Visited) const4244 MachineInstr *PPCInstrInfo::findLoopInstr(
4245 MachineBasicBlock &PreHeader,
4246 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
4247
4248 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
4249
4250 // The loop set-up instruction should be in preheader
4251 for (auto &I : PreHeader.instrs())
4252 if (I.getOpcode() == LOOPi)
4253 return &I;
4254 return nullptr;
4255 }
4256
4257 // Return true if get the base operand, byte offset of an instruction and the
4258 // memory width. Width is the size of memory that is being loaded/stored.
getMemOperandWithOffsetWidth(const MachineInstr & LdSt,const MachineOperand * & BaseReg,int64_t & Offset,unsigned & Width,const TargetRegisterInfo * TRI) const4259 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
4260 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
4261 unsigned &Width, const TargetRegisterInfo *TRI) const {
4262 if (!LdSt.mayLoadOrStore())
4263 return false;
4264
4265 // Handle only loads/stores with base register followed by immediate offset.
4266 if (LdSt.getNumExplicitOperands() != 3)
4267 return false;
4268 if (!LdSt.getOperand(1).isImm() || !LdSt.getOperand(2).isReg())
4269 return false;
4270
4271 if (!LdSt.hasOneMemOperand())
4272 return false;
4273
4274 Width = (*LdSt.memoperands_begin())->getSize();
4275 Offset = LdSt.getOperand(1).getImm();
4276 BaseReg = &LdSt.getOperand(2);
4277 return true;
4278 }
4279
areMemAccessesTriviallyDisjoint(const MachineInstr & MIa,const MachineInstr & MIb) const4280 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
4281 const MachineInstr &MIa, const MachineInstr &MIb) const {
4282 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
4283 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
4284
4285 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
4286 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4287 return false;
4288
4289 // Retrieve the base register, offset from the base register and width. Width
4290 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
4291 // base registers are identical, and the offset of a lower memory access +
4292 // the width doesn't overlap the offset of a higher memory access,
4293 // then the memory accesses are different.
4294 const TargetRegisterInfo *TRI = &getRegisterInfo();
4295 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
4296 int64_t OffsetA = 0, OffsetB = 0;
4297 unsigned int WidthA = 0, WidthB = 0;
4298 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
4299 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
4300 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
4301 int LowOffset = std::min(OffsetA, OffsetB);
4302 int HighOffset = std::max(OffsetA, OffsetB);
4303 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4304 if (LowOffset + LowWidth <= HighOffset)
4305 return true;
4306 }
4307 }
4308 return false;
4309 }
4310