1 //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling and other late optimizations. This
11 // pass should be run after register allocation but before the post-regalloc
12 // scheduling pass.
13 //
14 //===----------------------------------------------------------------------===//
15
16 #include "AArch64ExpandImm.h"
17 #include "AArch64InstrInfo.h"
18 #include "AArch64MachineFunctionInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineBasicBlock.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineInstr.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineOperand.h"
31 #include "llvm/CodeGen/TargetSubtargetInfo.h"
32 #include "llvm/IR/DebugLoc.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/Pass.h"
35 #include "llvm/Support/CodeGen.h"
36 #include "llvm/Support/MathExtras.h"
37 #include "llvm/Target/TargetMachine.h"
38 #include <cassert>
39 #include <cstdint>
40 #include <iterator>
41 #include <limits>
42 #include <utility>
43
44 using namespace llvm;
45
46 #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
47
48 namespace {
49
50 class AArch64ExpandPseudo : public MachineFunctionPass {
51 public:
52 const AArch64InstrInfo *TII;
53
54 static char ID;
55
AArch64ExpandPseudo()56 AArch64ExpandPseudo() : MachineFunctionPass(ID) {
57 initializeAArch64ExpandPseudoPass(*PassRegistry::getPassRegistry());
58 }
59
60 bool runOnMachineFunction(MachineFunction &Fn) override;
61
getPassName() const62 StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
63
64 private:
65 bool expandMBB(MachineBasicBlock &MBB);
66 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67 MachineBasicBlock::iterator &NextMBBI);
68 bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
69 unsigned BitSize);
70
71 bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73 unsigned ExtendImm, unsigned ZeroReg,
74 MachineBasicBlock::iterator &NextMBBI);
75 bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76 MachineBasicBlock::iterator MBBI,
77 MachineBasicBlock::iterator &NextMBBI);
78 bool expandSetTagLoop(MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI,
80 MachineBasicBlock::iterator &NextMBBI);
81 };
82
83 } // end anonymous namespace
84
85 char AArch64ExpandPseudo::ID = 0;
86
87 INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",
88 AARCH64_EXPAND_PSEUDO_NAME, false, false)
89
90 /// Transfer implicit operands on the pseudo instruction to the
91 /// instructions created from the expansion.
transferImpOps(MachineInstr & OldMI,MachineInstrBuilder & UseMI,MachineInstrBuilder & DefMI)92 static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
93 MachineInstrBuilder &DefMI) {
94 const MCInstrDesc &Desc = OldMI.getDesc();
95 for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands(); i != e;
96 ++i) {
97 const MachineOperand &MO = OldMI.getOperand(i);
98 assert(MO.isReg() && MO.getReg());
99 if (MO.isUse())
100 UseMI.add(MO);
101 else
102 DefMI.add(MO);
103 }
104 }
105
106 /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
107 /// real move-immediate instructions to synthesize the immediate.
expandMOVImm(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,unsigned BitSize)108 bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB,
109 MachineBasicBlock::iterator MBBI,
110 unsigned BitSize) {
111 MachineInstr &MI = *MBBI;
112 unsigned DstReg = MI.getOperand(0).getReg();
113 uint64_t Imm = MI.getOperand(1).getImm();
114
115 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
116 // Useless def, and we don't want to risk creating an invalid ORR (which
117 // would really write to sp).
118 MI.eraseFromParent();
119 return true;
120 }
121
122 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
123 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
124 assert(Insn.size() != 0);
125
126 SmallVector<MachineInstrBuilder, 4> MIBS;
127 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
128 bool LastItem = std::next(I) == E;
129 switch (I->Opcode)
130 {
131 default: llvm_unreachable("unhandled!"); break;
132
133 case AArch64::ORRWri:
134 case AArch64::ORRXri:
135 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
136 .add(MI.getOperand(0))
137 .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
138 .addImm(I->Op2));
139 break;
140 case AArch64::MOVNWi:
141 case AArch64::MOVNXi:
142 case AArch64::MOVZWi:
143 case AArch64::MOVZXi: {
144 bool DstIsDead = MI.getOperand(0).isDead();
145 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
146 .addReg(DstReg, RegState::Define |
147 getDeadRegState(DstIsDead && LastItem))
148 .addImm(I->Op1)
149 .addImm(I->Op2));
150 } break;
151 case AArch64::MOVKWi:
152 case AArch64::MOVKXi: {
153 unsigned DstReg = MI.getOperand(0).getReg();
154 bool DstIsDead = MI.getOperand(0).isDead();
155 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
156 .addReg(DstReg,
157 RegState::Define |
158 getDeadRegState(DstIsDead && LastItem))
159 .addReg(DstReg)
160 .addImm(I->Op1)
161 .addImm(I->Op2));
162 } break;
163 }
164 }
165 transferImpOps(MI, MIBS.front(), MIBS.back());
166 MI.eraseFromParent();
167 return true;
168 }
169
expandCMP_SWAP(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,unsigned LdarOp,unsigned StlrOp,unsigned CmpOp,unsigned ExtendImm,unsigned ZeroReg,MachineBasicBlock::iterator & NextMBBI)170 bool AArch64ExpandPseudo::expandCMP_SWAP(
171 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
172 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
173 MachineBasicBlock::iterator &NextMBBI) {
174 MachineInstr &MI = *MBBI;
175 DebugLoc DL = MI.getDebugLoc();
176 const MachineOperand &Dest = MI.getOperand(0);
177 unsigned StatusReg = MI.getOperand(1).getReg();
178 bool StatusDead = MI.getOperand(1).isDead();
179 // Duplicating undef operands into 2 instructions does not guarantee the same
180 // value on both; However undef should be replaced by xzr anyway.
181 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
182 unsigned AddrReg = MI.getOperand(2).getReg();
183 unsigned DesiredReg = MI.getOperand(3).getReg();
184 unsigned NewReg = MI.getOperand(4).getReg();
185
186 MachineFunction *MF = MBB.getParent();
187 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
188 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
189 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
190
191 MF->insert(++MBB.getIterator(), LoadCmpBB);
192 MF->insert(++LoadCmpBB->getIterator(), StoreBB);
193 MF->insert(++StoreBB->getIterator(), DoneBB);
194
195 // .Lloadcmp:
196 // mov wStatus, 0
197 // ldaxr xDest, [xAddr]
198 // cmp xDest, xDesired
199 // b.ne .Ldone
200 if (!StatusDead)
201 BuildMI(LoadCmpBB, DL, TII->get(AArch64::MOVZWi), StatusReg)
202 .addImm(0).addImm(0);
203 BuildMI(LoadCmpBB, DL, TII->get(LdarOp), Dest.getReg())
204 .addReg(AddrReg);
205 BuildMI(LoadCmpBB, DL, TII->get(CmpOp), ZeroReg)
206 .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
207 .addReg(DesiredReg)
208 .addImm(ExtendImm);
209 BuildMI(LoadCmpBB, DL, TII->get(AArch64::Bcc))
210 .addImm(AArch64CC::NE)
211 .addMBB(DoneBB)
212 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
213 LoadCmpBB->addSuccessor(DoneBB);
214 LoadCmpBB->addSuccessor(StoreBB);
215
216 // .Lstore:
217 // stlxr wStatus, xNew, [xAddr]
218 // cbnz wStatus, .Lloadcmp
219 BuildMI(StoreBB, DL, TII->get(StlrOp), StatusReg)
220 .addReg(NewReg)
221 .addReg(AddrReg);
222 BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
223 .addReg(StatusReg, getKillRegState(StatusDead))
224 .addMBB(LoadCmpBB);
225 StoreBB->addSuccessor(LoadCmpBB);
226 StoreBB->addSuccessor(DoneBB);
227
228 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
229 DoneBB->transferSuccessors(&MBB);
230
231 MBB.addSuccessor(LoadCmpBB);
232
233 NextMBBI = MBB.end();
234 MI.eraseFromParent();
235
236 // Recompute livein lists.
237 LivePhysRegs LiveRegs;
238 computeAndAddLiveIns(LiveRegs, *DoneBB);
239 computeAndAddLiveIns(LiveRegs, *StoreBB);
240 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
241 // Do an extra pass around the loop to get loop carried registers right.
242 StoreBB->clearLiveIns();
243 computeAndAddLiveIns(LiveRegs, *StoreBB);
244 LoadCmpBB->clearLiveIns();
245 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
246
247 return true;
248 }
249
expandCMP_SWAP_128(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)250 bool AArch64ExpandPseudo::expandCMP_SWAP_128(
251 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
252 MachineBasicBlock::iterator &NextMBBI) {
253 MachineInstr &MI = *MBBI;
254 DebugLoc DL = MI.getDebugLoc();
255 MachineOperand &DestLo = MI.getOperand(0);
256 MachineOperand &DestHi = MI.getOperand(1);
257 unsigned StatusReg = MI.getOperand(2).getReg();
258 bool StatusDead = MI.getOperand(2).isDead();
259 // Duplicating undef operands into 2 instructions does not guarantee the same
260 // value on both; However undef should be replaced by xzr anyway.
261 assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
262 unsigned AddrReg = MI.getOperand(3).getReg();
263 unsigned DesiredLoReg = MI.getOperand(4).getReg();
264 unsigned DesiredHiReg = MI.getOperand(5).getReg();
265 unsigned NewLoReg = MI.getOperand(6).getReg();
266 unsigned NewHiReg = MI.getOperand(7).getReg();
267
268 MachineFunction *MF = MBB.getParent();
269 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
270 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
271 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
272
273 MF->insert(++MBB.getIterator(), LoadCmpBB);
274 MF->insert(++LoadCmpBB->getIterator(), StoreBB);
275 MF->insert(++StoreBB->getIterator(), DoneBB);
276
277 // .Lloadcmp:
278 // ldaxp xDestLo, xDestHi, [xAddr]
279 // cmp xDestLo, xDesiredLo
280 // sbcs xDestHi, xDesiredHi
281 // b.ne .Ldone
282 BuildMI(LoadCmpBB, DL, TII->get(AArch64::LDAXPX))
283 .addReg(DestLo.getReg(), RegState::Define)
284 .addReg(DestHi.getReg(), RegState::Define)
285 .addReg(AddrReg);
286 BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
287 .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
288 .addReg(DesiredLoReg)
289 .addImm(0);
290 BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
291 .addUse(AArch64::WZR)
292 .addUse(AArch64::WZR)
293 .addImm(AArch64CC::EQ);
294 BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
295 .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
296 .addReg(DesiredHiReg)
297 .addImm(0);
298 BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
299 .addUse(StatusReg, RegState::Kill)
300 .addUse(StatusReg, RegState::Kill)
301 .addImm(AArch64CC::EQ);
302 BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
303 .addUse(StatusReg, getKillRegState(StatusDead))
304 .addMBB(DoneBB);
305 LoadCmpBB->addSuccessor(DoneBB);
306 LoadCmpBB->addSuccessor(StoreBB);
307
308 // .Lstore:
309 // stlxp wStatus, xNewLo, xNewHi, [xAddr]
310 // cbnz wStatus, .Lloadcmp
311 BuildMI(StoreBB, DL, TII->get(AArch64::STLXPX), StatusReg)
312 .addReg(NewLoReg)
313 .addReg(NewHiReg)
314 .addReg(AddrReg);
315 BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
316 .addReg(StatusReg, getKillRegState(StatusDead))
317 .addMBB(LoadCmpBB);
318 StoreBB->addSuccessor(LoadCmpBB);
319 StoreBB->addSuccessor(DoneBB);
320
321 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
322 DoneBB->transferSuccessors(&MBB);
323
324 MBB.addSuccessor(LoadCmpBB);
325
326 NextMBBI = MBB.end();
327 MI.eraseFromParent();
328
329 // Recompute liveness bottom up.
330 LivePhysRegs LiveRegs;
331 computeAndAddLiveIns(LiveRegs, *DoneBB);
332 computeAndAddLiveIns(LiveRegs, *StoreBB);
333 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
334 // Do an extra pass in the loop to get the loop carried dependencies right.
335 StoreBB->clearLiveIns();
336 computeAndAddLiveIns(LiveRegs, *StoreBB);
337 LoadCmpBB->clearLiveIns();
338 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
339
340 return true;
341 }
342
expandSetTagLoop(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)343 bool AArch64ExpandPseudo::expandSetTagLoop(
344 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
345 MachineBasicBlock::iterator &NextMBBI) {
346 MachineInstr &MI = *MBBI;
347 DebugLoc DL = MI.getDebugLoc();
348 Register SizeReg = MI.getOperand(2).getReg();
349 Register AddressReg = MI.getOperand(3).getReg();
350
351 MachineFunction *MF = MBB.getParent();
352
353 bool ZeroData = MI.getOpcode() == AArch64::STZGloop;
354 const unsigned OpCode =
355 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
356
357 auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
358 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
359
360 MF->insert(++MBB.getIterator(), LoopBB);
361 MF->insert(++LoopBB->getIterator(), DoneBB);
362
363 BuildMI(LoopBB, DL, TII->get(OpCode))
364 .addDef(AddressReg)
365 .addReg(AddressReg)
366 .addReg(AddressReg)
367 .addImm(2)
368 .cloneMemRefs(MI)
369 .setMIFlags(MI.getFlags());
370 BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri))
371 .addDef(SizeReg)
372 .addReg(SizeReg)
373 .addImm(16 * 2)
374 .addImm(0);
375 BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB);
376
377 LoopBB->addSuccessor(LoopBB);
378 LoopBB->addSuccessor(DoneBB);
379
380 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
381 DoneBB->transferSuccessors(&MBB);
382
383 MBB.addSuccessor(LoopBB);
384
385 NextMBBI = MBB.end();
386 MI.eraseFromParent();
387 // Recompute liveness bottom up.
388 LivePhysRegs LiveRegs;
389 computeAndAddLiveIns(LiveRegs, *DoneBB);
390 computeAndAddLiveIns(LiveRegs, *LoopBB);
391 // Do an extra pass in the loop to get the loop carried dependencies right.
392 // FIXME: is this necessary?
393 LoopBB->clearLiveIns();
394 computeAndAddLiveIns(LiveRegs, *LoopBB);
395 DoneBB->clearLiveIns();
396 computeAndAddLiveIns(LiveRegs, *DoneBB);
397
398 return true;
399 }
400
401 /// If MBBI references a pseudo instruction that should be expanded here,
402 /// do the expansion and return true. Otherwise return false.
expandMI(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)403 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
404 MachineBasicBlock::iterator MBBI,
405 MachineBasicBlock::iterator &NextMBBI) {
406 MachineInstr &MI = *MBBI;
407 unsigned Opcode = MI.getOpcode();
408 switch (Opcode) {
409 default:
410 break;
411
412 case AArch64::ADDWrr:
413 case AArch64::SUBWrr:
414 case AArch64::ADDXrr:
415 case AArch64::SUBXrr:
416 case AArch64::ADDSWrr:
417 case AArch64::SUBSWrr:
418 case AArch64::ADDSXrr:
419 case AArch64::SUBSXrr:
420 case AArch64::ANDWrr:
421 case AArch64::ANDXrr:
422 case AArch64::BICWrr:
423 case AArch64::BICXrr:
424 case AArch64::ANDSWrr:
425 case AArch64::ANDSXrr:
426 case AArch64::BICSWrr:
427 case AArch64::BICSXrr:
428 case AArch64::EONWrr:
429 case AArch64::EONXrr:
430 case AArch64::EORWrr:
431 case AArch64::EORXrr:
432 case AArch64::ORNWrr:
433 case AArch64::ORNXrr:
434 case AArch64::ORRWrr:
435 case AArch64::ORRXrr: {
436 unsigned Opcode;
437 switch (MI.getOpcode()) {
438 default:
439 return false;
440 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
441 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
442 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
443 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
444 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
445 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
446 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
447 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
448 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
449 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
450 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
451 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
452 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
453 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
454 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
455 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
456 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
457 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
458 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
459 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
460 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
461 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
462 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
463 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
464 }
465 MachineInstrBuilder MIB1 =
466 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode),
467 MI.getOperand(0).getReg())
468 .add(MI.getOperand(1))
469 .add(MI.getOperand(2))
470 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
471 transferImpOps(MI, MIB1, MIB1);
472 MI.eraseFromParent();
473 return true;
474 }
475
476 case AArch64::LOADgot: {
477 MachineFunction *MF = MBB.getParent();
478 unsigned DstReg = MI.getOperand(0).getReg();
479 const MachineOperand &MO1 = MI.getOperand(1);
480 unsigned Flags = MO1.getTargetFlags();
481
482 if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
483 // Tiny codemodel expand to LDR
484 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
485 TII->get(AArch64::LDRXl), DstReg);
486
487 if (MO1.isGlobal()) {
488 MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);
489 } else if (MO1.isSymbol()) {
490 MIB.addExternalSymbol(MO1.getSymbolName(), Flags);
491 } else {
492 assert(MO1.isCPI() &&
493 "Only expect globals, externalsymbols, or constant pools");
494 MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);
495 }
496 } else {
497 // Small codemodel expand into ADRP + LDR.
498 MachineInstrBuilder MIB1 =
499 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);
500 MachineInstrBuilder MIB2 =
501 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui))
502 .add(MI.getOperand(0))
503 .addReg(DstReg);
504
505 if (MO1.isGlobal()) {
506 MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);
507 MIB2.addGlobalAddress(MO1.getGlobal(), 0,
508 Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
509 } else if (MO1.isSymbol()) {
510 MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE);
511 MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |
512 AArch64II::MO_PAGEOFF |
513 AArch64II::MO_NC);
514 } else {
515 assert(MO1.isCPI() &&
516 "Only expect globals, externalsymbols, or constant pools");
517 MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
518 Flags | AArch64II::MO_PAGE);
519 MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
520 Flags | AArch64II::MO_PAGEOFF |
521 AArch64II::MO_NC);
522 }
523
524 transferImpOps(MI, MIB1, MIB2);
525 }
526 MI.eraseFromParent();
527 return true;
528 }
529
530 case AArch64::MOVaddr:
531 case AArch64::MOVaddrJT:
532 case AArch64::MOVaddrCP:
533 case AArch64::MOVaddrBA:
534 case AArch64::MOVaddrTLS:
535 case AArch64::MOVaddrEXT: {
536 // Expand into ADRP + ADD.
537 unsigned DstReg = MI.getOperand(0).getReg();
538 MachineInstrBuilder MIB1 =
539 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
540 .add(MI.getOperand(1));
541
542 MachineInstrBuilder MIB2 =
543 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
544 .add(MI.getOperand(0))
545 .addReg(DstReg)
546 .add(MI.getOperand(2))
547 .addImm(0);
548
549 transferImpOps(MI, MIB1, MIB2);
550 MI.eraseFromParent();
551 return true;
552 }
553 case AArch64::ADDlowTLS:
554 // Produce a plain ADD
555 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
556 .add(MI.getOperand(0))
557 .add(MI.getOperand(1))
558 .add(MI.getOperand(2))
559 .addImm(0);
560 MI.eraseFromParent();
561 return true;
562
563 case AArch64::MOVbaseTLS: {
564 unsigned DstReg = MI.getOperand(0).getReg();
565 auto SysReg = AArch64SysReg::TPIDR_EL0;
566 MachineFunction *MF = MBB.getParent();
567 if (MF->getTarget().getTargetTriple().isOSFuchsia() &&
568 MF->getTarget().getCodeModel() == CodeModel::Kernel)
569 SysReg = AArch64SysReg::TPIDR_EL1;
570 else if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
571 SysReg = AArch64SysReg::TPIDR_EL3;
572 else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
573 SysReg = AArch64SysReg::TPIDR_EL2;
574 else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
575 SysReg = AArch64SysReg::TPIDR_EL1;
576 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)
577 .addImm(SysReg);
578 MI.eraseFromParent();
579 return true;
580 }
581
582 case AArch64::MOVi32imm:
583 return expandMOVImm(MBB, MBBI, 32);
584 case AArch64::MOVi64imm:
585 return expandMOVImm(MBB, MBBI, 64);
586 case AArch64::RET_ReallyLR: {
587 // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
588 // function and missing live-ins. We are fine in practice because callee
589 // saved register handling ensures the register value is restored before
590 // RET, but we need the undef flag here to appease the MachineVerifier
591 // liveness checks.
592 MachineInstrBuilder MIB =
593 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))
594 .addReg(AArch64::LR, RegState::Undef);
595 transferImpOps(MI, MIB, MIB);
596 MI.eraseFromParent();
597 return true;
598 }
599 case AArch64::CMP_SWAP_8:
600 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
601 AArch64::SUBSWrx,
602 AArch64_AM::getArithExtendImm(AArch64_AM::UXTB, 0),
603 AArch64::WZR, NextMBBI);
604 case AArch64::CMP_SWAP_16:
605 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
606 AArch64::SUBSWrx,
607 AArch64_AM::getArithExtendImm(AArch64_AM::UXTH, 0),
608 AArch64::WZR, NextMBBI);
609 case AArch64::CMP_SWAP_32:
610 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
611 AArch64::SUBSWrs,
612 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
613 AArch64::WZR, NextMBBI);
614 case AArch64::CMP_SWAP_64:
615 return expandCMP_SWAP(MBB, MBBI,
616 AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
617 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
618 AArch64::XZR, NextMBBI);
619 case AArch64::CMP_SWAP_128:
620 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
621
622 case AArch64::AESMCrrTied:
623 case AArch64::AESIMCrrTied: {
624 MachineInstrBuilder MIB =
625 BuildMI(MBB, MBBI, MI.getDebugLoc(),
626 TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
627 AArch64::AESIMCrr))
628 .add(MI.getOperand(0))
629 .add(MI.getOperand(1));
630 transferImpOps(MI, MIB, MIB);
631 MI.eraseFromParent();
632 return true;
633 }
634 case AArch64::IRGstack: {
635 MachineFunction &MF = *MBB.getParent();
636 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
637 const AArch64FrameLowering *TFI =
638 MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
639
640 // IRG does not allow immediate offset. getTaggedBasePointerOffset should
641 // almost always point to SP-after-prologue; if not, emit a longer
642 // instruction sequence.
643 int BaseOffset = -AFI->getTaggedBasePointerOffset();
644 unsigned FrameReg;
645 int FrameRegOffset = TFI->resolveFrameOffsetReference(
646 MF, BaseOffset, false /*isFixed*/, FrameReg, /*PreferFP=*/false,
647 /*ForSimm=*/true);
648 Register SrcReg = FrameReg;
649 if (FrameRegOffset != 0) {
650 // Use output register as temporary.
651 SrcReg = MI.getOperand(0).getReg();
652 emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,
653 FrameRegOffset, TII);
654 }
655 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))
656 .add(MI.getOperand(0))
657 .addUse(SrcReg)
658 .add(MI.getOperand(2));
659 MI.eraseFromParent();
660 return true;
661 }
662 case AArch64::TAGPstack: {
663 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDG))
664 .add(MI.getOperand(0))
665 .add(MI.getOperand(1))
666 .add(MI.getOperand(2))
667 .add(MI.getOperand(4));
668 MI.eraseFromParent();
669 return true;
670 }
671 case AArch64::STGloop:
672 case AArch64::STZGloop:
673 return expandSetTagLoop(MBB, MBBI, NextMBBI);
674 }
675 return false;
676 }
677
678 /// Iterate over the instructions in basic block MBB and expand any
679 /// pseudo instructions. Return true if anything was modified.
expandMBB(MachineBasicBlock & MBB)680 bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
681 bool Modified = false;
682
683 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
684 while (MBBI != E) {
685 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
686 Modified |= expandMI(MBB, MBBI, NMBBI);
687 MBBI = NMBBI;
688 }
689
690 return Modified;
691 }
692
runOnMachineFunction(MachineFunction & MF)693 bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
694 TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
695
696 bool Modified = false;
697 for (auto &MBB : MF)
698 Modified |= expandMBB(MBB);
699 return Modified;
700 }
701
702 /// Returns an instance of the pseudo instruction expansion pass.
createAArch64ExpandPseudoPass()703 FunctionPass *llvm::createAArch64ExpandPseudoPass() {
704 return new AArch64ExpandPseudo();
705 }
706