1 //===- RDFGraph.cpp -------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Target-independent, SSA-based data flow graph for register data flow (RDF).
10 //
11 #include "llvm/ADT/BitVector.h"
12 #include "llvm/ADT/STLExtras.h"
13 #include "llvm/ADT/SetVector.h"
14 #include "llvm/CodeGen/MachineBasicBlock.h"
15 #include "llvm/CodeGen/MachineDominanceFrontier.h"
16 #include "llvm/CodeGen/MachineDominators.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/MachineOperand.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/RDFGraph.h"
22 #include "llvm/CodeGen/RDFRegisters.h"
23 #include "llvm/CodeGen/TargetInstrInfo.h"
24 #include "llvm/CodeGen/TargetLowering.h"
25 #include "llvm/CodeGen/TargetRegisterInfo.h"
26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/MC/LaneBitmask.h"
29 #include "llvm/MC/MCInstrDesc.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/ErrorHandling.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include <algorithm>
35 #include <cassert>
36 #include <cstdint>
37 #include <cstring>
38 #include <iterator>
39 #include <set>
40 #include <utility>
41 #include <vector>
42
43 using namespace llvm;
44 using namespace rdf;
45
46 // Printing functions. Have them here first, so that the rest of the code
47 // can use them.
48 namespace llvm {
49 namespace rdf {
50
operator <<(raw_ostream & OS,const PrintLaneMaskOpt & P)51 raw_ostream &operator<< (raw_ostream &OS, const PrintLaneMaskOpt &P) {
52 if (!P.Mask.all())
53 OS << ':' << PrintLaneMask(P.Mask);
54 return OS;
55 }
56
operator <<(raw_ostream & OS,const Print<RegisterRef> & P)57 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterRef> &P) {
58 auto &TRI = P.G.getTRI();
59 if (P.Obj.Reg > 0 && P.Obj.Reg < TRI.getNumRegs())
60 OS << TRI.getName(P.Obj.Reg);
61 else
62 OS << '#' << P.Obj.Reg;
63 OS << PrintLaneMaskOpt(P.Obj.Mask);
64 return OS;
65 }
66
operator <<(raw_ostream & OS,const Print<NodeId> & P)67 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeId> &P) {
68 auto NA = P.G.addr<NodeBase*>(P.Obj);
69 uint16_t Attrs = NA.Addr->getAttrs();
70 uint16_t Kind = NodeAttrs::kind(Attrs);
71 uint16_t Flags = NodeAttrs::flags(Attrs);
72 switch (NodeAttrs::type(Attrs)) {
73 case NodeAttrs::Code:
74 switch (Kind) {
75 case NodeAttrs::Func: OS << 'f'; break;
76 case NodeAttrs::Block: OS << 'b'; break;
77 case NodeAttrs::Stmt: OS << 's'; break;
78 case NodeAttrs::Phi: OS << 'p'; break;
79 default: OS << "c?"; break;
80 }
81 break;
82 case NodeAttrs::Ref:
83 if (Flags & NodeAttrs::Undef)
84 OS << '/';
85 if (Flags & NodeAttrs::Dead)
86 OS << '\\';
87 if (Flags & NodeAttrs::Preserving)
88 OS << '+';
89 if (Flags & NodeAttrs::Clobbering)
90 OS << '~';
91 switch (Kind) {
92 case NodeAttrs::Use: OS << 'u'; break;
93 case NodeAttrs::Def: OS << 'd'; break;
94 case NodeAttrs::Block: OS << 'b'; break;
95 default: OS << "r?"; break;
96 }
97 break;
98 default:
99 OS << '?';
100 break;
101 }
102 OS << P.Obj;
103 if (Flags & NodeAttrs::Shadow)
104 OS << '"';
105 return OS;
106 }
107
printRefHeader(raw_ostream & OS,const NodeAddr<RefNode * > RA,const DataFlowGraph & G)108 static void printRefHeader(raw_ostream &OS, const NodeAddr<RefNode*> RA,
109 const DataFlowGraph &G) {
110 OS << Print<NodeId>(RA.Id, G) << '<'
111 << Print<RegisterRef>(RA.Addr->getRegRef(G), G) << '>';
112 if (RA.Addr->getFlags() & NodeAttrs::Fixed)
113 OS << '!';
114 }
115
operator <<(raw_ostream & OS,const Print<NodeAddr<DefNode * >> & P)116 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<DefNode*>> &P) {
117 printRefHeader(OS, P.Obj, P.G);
118 OS << '(';
119 if (NodeId N = P.Obj.Addr->getReachingDef())
120 OS << Print<NodeId>(N, P.G);
121 OS << ',';
122 if (NodeId N = P.Obj.Addr->getReachedDef())
123 OS << Print<NodeId>(N, P.G);
124 OS << ',';
125 if (NodeId N = P.Obj.Addr->getReachedUse())
126 OS << Print<NodeId>(N, P.G);
127 OS << "):";
128 if (NodeId N = P.Obj.Addr->getSibling())
129 OS << Print<NodeId>(N, P.G);
130 return OS;
131 }
132
operator <<(raw_ostream & OS,const Print<NodeAddr<UseNode * >> & P)133 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<UseNode*>> &P) {
134 printRefHeader(OS, P.Obj, P.G);
135 OS << '(';
136 if (NodeId N = P.Obj.Addr->getReachingDef())
137 OS << Print<NodeId>(N, P.G);
138 OS << "):";
139 if (NodeId N = P.Obj.Addr->getSibling())
140 OS << Print<NodeId>(N, P.G);
141 return OS;
142 }
143
operator <<(raw_ostream & OS,const Print<NodeAddr<PhiUseNode * >> & P)144 raw_ostream &operator<< (raw_ostream &OS,
145 const Print<NodeAddr<PhiUseNode*>> &P) {
146 printRefHeader(OS, P.Obj, P.G);
147 OS << '(';
148 if (NodeId N = P.Obj.Addr->getReachingDef())
149 OS << Print<NodeId>(N, P.G);
150 OS << ',';
151 if (NodeId N = P.Obj.Addr->getPredecessor())
152 OS << Print<NodeId>(N, P.G);
153 OS << "):";
154 if (NodeId N = P.Obj.Addr->getSibling())
155 OS << Print<NodeId>(N, P.G);
156 return OS;
157 }
158
operator <<(raw_ostream & OS,const Print<NodeAddr<RefNode * >> & P)159 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<RefNode*>> &P) {
160 switch (P.Obj.Addr->getKind()) {
161 case NodeAttrs::Def:
162 OS << PrintNode<DefNode*>(P.Obj, P.G);
163 break;
164 case NodeAttrs::Use:
165 if (P.Obj.Addr->getFlags() & NodeAttrs::PhiRef)
166 OS << PrintNode<PhiUseNode*>(P.Obj, P.G);
167 else
168 OS << PrintNode<UseNode*>(P.Obj, P.G);
169 break;
170 }
171 return OS;
172 }
173
operator <<(raw_ostream & OS,const Print<NodeList> & P)174 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeList> &P) {
175 unsigned N = P.Obj.size();
176 for (auto I : P.Obj) {
177 OS << Print<NodeId>(I.Id, P.G);
178 if (--N)
179 OS << ' ';
180 }
181 return OS;
182 }
183
operator <<(raw_ostream & OS,const Print<NodeSet> & P)184 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeSet> &P) {
185 unsigned N = P.Obj.size();
186 for (auto I : P.Obj) {
187 OS << Print<NodeId>(I, P.G);
188 if (--N)
189 OS << ' ';
190 }
191 return OS;
192 }
193
194 namespace {
195
196 template <typename T>
197 struct PrintListV {
PrintListVllvm::rdf::__anon346a69eb0111::PrintListV198 PrintListV(const NodeList &L, const DataFlowGraph &G) : List(L), G(G) {}
199
200 using Type = T;
201 const NodeList &List;
202 const DataFlowGraph &G;
203 };
204
205 template <typename T>
operator <<(raw_ostream & OS,const PrintListV<T> & P)206 raw_ostream &operator<< (raw_ostream &OS, const PrintListV<T> &P) {
207 unsigned N = P.List.size();
208 for (NodeAddr<T> A : P.List) {
209 OS << PrintNode<T>(A, P.G);
210 if (--N)
211 OS << ", ";
212 }
213 return OS;
214 }
215
216 } // end anonymous namespace
217
operator <<(raw_ostream & OS,const Print<NodeAddr<PhiNode * >> & P)218 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<PhiNode*>> &P) {
219 OS << Print<NodeId>(P.Obj.Id, P.G) << ": phi ["
220 << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
221 return OS;
222 }
223
operator <<(raw_ostream & OS,const Print<NodeAddr<StmtNode * >> & P)224 raw_ostream &operator<<(raw_ostream &OS, const Print<NodeAddr<StmtNode *>> &P) {
225 const MachineInstr &MI = *P.Obj.Addr->getCode();
226 unsigned Opc = MI.getOpcode();
227 OS << Print<NodeId>(P.Obj.Id, P.G) << ": " << P.G.getTII().getName(Opc);
228 // Print the target for calls and branches (for readability).
229 if (MI.isCall() || MI.isBranch()) {
230 MachineInstr::const_mop_iterator T =
231 llvm::find_if(MI.operands(),
232 [] (const MachineOperand &Op) -> bool {
233 return Op.isMBB() || Op.isGlobal() || Op.isSymbol();
234 });
235 if (T != MI.operands_end()) {
236 OS << ' ';
237 if (T->isMBB())
238 OS << printMBBReference(*T->getMBB());
239 else if (T->isGlobal())
240 OS << T->getGlobal()->getName();
241 else if (T->isSymbol())
242 OS << T->getSymbolName();
243 }
244 }
245 OS << " [" << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
246 return OS;
247 }
248
operator <<(raw_ostream & OS,const Print<NodeAddr<InstrNode * >> & P)249 raw_ostream &operator<< (raw_ostream &OS,
250 const Print<NodeAddr<InstrNode*>> &P) {
251 switch (P.Obj.Addr->getKind()) {
252 case NodeAttrs::Phi:
253 OS << PrintNode<PhiNode*>(P.Obj, P.G);
254 break;
255 case NodeAttrs::Stmt:
256 OS << PrintNode<StmtNode*>(P.Obj, P.G);
257 break;
258 default:
259 OS << "instr? " << Print<NodeId>(P.Obj.Id, P.G);
260 break;
261 }
262 return OS;
263 }
264
operator <<(raw_ostream & OS,const Print<NodeAddr<BlockNode * >> & P)265 raw_ostream &operator<< (raw_ostream &OS,
266 const Print<NodeAddr<BlockNode*>> &P) {
267 MachineBasicBlock *BB = P.Obj.Addr->getCode();
268 unsigned NP = BB->pred_size();
269 std::vector<int> Ns;
270 auto PrintBBs = [&OS] (std::vector<int> Ns) -> void {
271 unsigned N = Ns.size();
272 for (int I : Ns) {
273 OS << "%bb." << I;
274 if (--N)
275 OS << ", ";
276 }
277 };
278
279 OS << Print<NodeId>(P.Obj.Id, P.G) << ": --- " << printMBBReference(*BB)
280 << " --- preds(" << NP << "): ";
281 for (MachineBasicBlock *B : BB->predecessors())
282 Ns.push_back(B->getNumber());
283 PrintBBs(Ns);
284
285 unsigned NS = BB->succ_size();
286 OS << " succs(" << NS << "): ";
287 Ns.clear();
288 for (MachineBasicBlock *B : BB->successors())
289 Ns.push_back(B->getNumber());
290 PrintBBs(Ns);
291 OS << '\n';
292
293 for (auto I : P.Obj.Addr->members(P.G))
294 OS << PrintNode<InstrNode*>(I, P.G) << '\n';
295 return OS;
296 }
297
operator <<(raw_ostream & OS,const Print<NodeAddr<FuncNode * >> & P)298 raw_ostream &operator<<(raw_ostream &OS, const Print<NodeAddr<FuncNode *>> &P) {
299 OS << "DFG dump:[\n" << Print<NodeId>(P.Obj.Id, P.G) << ": Function: "
300 << P.Obj.Addr->getCode()->getName() << '\n';
301 for (auto I : P.Obj.Addr->members(P.G))
302 OS << PrintNode<BlockNode*>(I, P.G) << '\n';
303 OS << "]\n";
304 return OS;
305 }
306
operator <<(raw_ostream & OS,const Print<RegisterSet> & P)307 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterSet> &P) {
308 OS << '{';
309 for (auto I : P.Obj)
310 OS << ' ' << Print<RegisterRef>(I, P.G);
311 OS << " }";
312 return OS;
313 }
314
operator <<(raw_ostream & OS,const Print<RegisterAggr> & P)315 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterAggr> &P) {
316 P.Obj.print(OS);
317 return OS;
318 }
319
operator <<(raw_ostream & OS,const Print<DataFlowGraph::DefStack> & P)320 raw_ostream &operator<< (raw_ostream &OS,
321 const Print<DataFlowGraph::DefStack> &P) {
322 for (auto I = P.Obj.top(), E = P.Obj.bottom(); I != E; ) {
323 OS << Print<NodeId>(I->Id, P.G)
324 << '<' << Print<RegisterRef>(I->Addr->getRegRef(P.G), P.G) << '>';
325 I.down();
326 if (I != E)
327 OS << ' ';
328 }
329 return OS;
330 }
331
332 } // end namespace rdf
333 } // end namespace llvm
334
335 // Node allocation functions.
336 //
337 // Node allocator is like a slab memory allocator: it allocates blocks of
338 // memory in sizes that are multiples of the size of a node. Each block has
339 // the same size. Nodes are allocated from the currently active block, and
340 // when it becomes full, a new one is created.
341 // There is a mapping scheme between node id and its location in a block,
342 // and within that block is described in the header file.
343 //
startNewBlock()344 void NodeAllocator::startNewBlock() {
345 void *T = MemPool.Allocate(NodesPerBlock*NodeMemSize, NodeMemSize);
346 char *P = static_cast<char*>(T);
347 Blocks.push_back(P);
348 // Check if the block index is still within the allowed range, i.e. less
349 // than 2^N, where N is the number of bits in NodeId for the block index.
350 // BitsPerIndex is the number of bits per node index.
351 assert((Blocks.size() < ((size_t)1 << (8*sizeof(NodeId)-BitsPerIndex))) &&
352 "Out of bits for block index");
353 ActiveEnd = P;
354 }
355
needNewBlock()356 bool NodeAllocator::needNewBlock() {
357 if (Blocks.empty())
358 return true;
359
360 char *ActiveBegin = Blocks.back();
361 uint32_t Index = (ActiveEnd-ActiveBegin)/NodeMemSize;
362 return Index >= NodesPerBlock;
363 }
364
New()365 NodeAddr<NodeBase*> NodeAllocator::New() {
366 if (needNewBlock())
367 startNewBlock();
368
369 uint32_t ActiveB = Blocks.size()-1;
370 uint32_t Index = (ActiveEnd - Blocks[ActiveB])/NodeMemSize;
371 NodeAddr<NodeBase*> NA = { reinterpret_cast<NodeBase*>(ActiveEnd),
372 makeId(ActiveB, Index) };
373 ActiveEnd += NodeMemSize;
374 return NA;
375 }
376
id(const NodeBase * P) const377 NodeId NodeAllocator::id(const NodeBase *P) const {
378 uintptr_t A = reinterpret_cast<uintptr_t>(P);
379 for (unsigned i = 0, n = Blocks.size(); i != n; ++i) {
380 uintptr_t B = reinterpret_cast<uintptr_t>(Blocks[i]);
381 if (A < B || A >= B + NodesPerBlock*NodeMemSize)
382 continue;
383 uint32_t Idx = (A-B)/NodeMemSize;
384 return makeId(i, Idx);
385 }
386 llvm_unreachable("Invalid node address");
387 }
388
clear()389 void NodeAllocator::clear() {
390 MemPool.Reset();
391 Blocks.clear();
392 ActiveEnd = nullptr;
393 }
394
395 // Insert node NA after "this" in the circular chain.
append(NodeAddr<NodeBase * > NA)396 void NodeBase::append(NodeAddr<NodeBase*> NA) {
397 NodeId Nx = Next;
398 // If NA is already "next", do nothing.
399 if (Next != NA.Id) {
400 Next = NA.Id;
401 NA.Addr->Next = Nx;
402 }
403 }
404
405 // Fundamental node manipulator functions.
406
407 // Obtain the register reference from a reference node.
getRegRef(const DataFlowGraph & G) const408 RegisterRef RefNode::getRegRef(const DataFlowGraph &G) const {
409 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
410 if (NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef)
411 return G.unpack(Ref.PR);
412 assert(Ref.Op != nullptr);
413 return G.makeRegRef(*Ref.Op);
414 }
415
416 // Set the register reference in the reference node directly (for references
417 // in phi nodes).
setRegRef(RegisterRef RR,DataFlowGraph & G)418 void RefNode::setRegRef(RegisterRef RR, DataFlowGraph &G) {
419 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
420 assert(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef);
421 Ref.PR = G.pack(RR);
422 }
423
424 // Set the register reference in the reference node based on a machine
425 // operand (for references in statement nodes).
setRegRef(MachineOperand * Op,DataFlowGraph & G)426 void RefNode::setRegRef(MachineOperand *Op, DataFlowGraph &G) {
427 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
428 assert(!(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef));
429 (void)G;
430 Ref.Op = Op;
431 }
432
433 // Get the owner of a given reference node.
getOwner(const DataFlowGraph & G)434 NodeAddr<NodeBase*> RefNode::getOwner(const DataFlowGraph &G) {
435 NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
436
437 while (NA.Addr != this) {
438 if (NA.Addr->getType() == NodeAttrs::Code)
439 return NA;
440 NA = G.addr<NodeBase*>(NA.Addr->getNext());
441 }
442 llvm_unreachable("No owner in circular list");
443 }
444
445 // Connect the def node to the reaching def node.
linkToDef(NodeId Self,NodeAddr<DefNode * > DA)446 void DefNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
447 Ref.RD = DA.Id;
448 Ref.Sib = DA.Addr->getReachedDef();
449 DA.Addr->setReachedDef(Self);
450 }
451
452 // Connect the use node to the reaching def node.
linkToDef(NodeId Self,NodeAddr<DefNode * > DA)453 void UseNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
454 Ref.RD = DA.Id;
455 Ref.Sib = DA.Addr->getReachedUse();
456 DA.Addr->setReachedUse(Self);
457 }
458
459 // Get the first member of the code node.
getFirstMember(const DataFlowGraph & G) const460 NodeAddr<NodeBase*> CodeNode::getFirstMember(const DataFlowGraph &G) const {
461 if (Code.FirstM == 0)
462 return NodeAddr<NodeBase*>();
463 return G.addr<NodeBase*>(Code.FirstM);
464 }
465
466 // Get the last member of the code node.
getLastMember(const DataFlowGraph & G) const467 NodeAddr<NodeBase*> CodeNode::getLastMember(const DataFlowGraph &G) const {
468 if (Code.LastM == 0)
469 return NodeAddr<NodeBase*>();
470 return G.addr<NodeBase*>(Code.LastM);
471 }
472
473 // Add node NA at the end of the member list of the given code node.
addMember(NodeAddr<NodeBase * > NA,const DataFlowGraph & G)474 void CodeNode::addMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
475 NodeAddr<NodeBase*> ML = getLastMember(G);
476 if (ML.Id != 0) {
477 ML.Addr->append(NA);
478 } else {
479 Code.FirstM = NA.Id;
480 NodeId Self = G.id(this);
481 NA.Addr->setNext(Self);
482 }
483 Code.LastM = NA.Id;
484 }
485
486 // Add node NA after member node MA in the given code node.
addMemberAfter(NodeAddr<NodeBase * > MA,NodeAddr<NodeBase * > NA,const DataFlowGraph & G)487 void CodeNode::addMemberAfter(NodeAddr<NodeBase*> MA, NodeAddr<NodeBase*> NA,
488 const DataFlowGraph &G) {
489 MA.Addr->append(NA);
490 if (Code.LastM == MA.Id)
491 Code.LastM = NA.Id;
492 }
493
494 // Remove member node NA from the given code node.
removeMember(NodeAddr<NodeBase * > NA,const DataFlowGraph & G)495 void CodeNode::removeMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
496 NodeAddr<NodeBase*> MA = getFirstMember(G);
497 assert(MA.Id != 0);
498
499 // Special handling if the member to remove is the first member.
500 if (MA.Id == NA.Id) {
501 if (Code.LastM == MA.Id) {
502 // If it is the only member, set both first and last to 0.
503 Code.FirstM = Code.LastM = 0;
504 } else {
505 // Otherwise, advance the first member.
506 Code.FirstM = MA.Addr->getNext();
507 }
508 return;
509 }
510
511 while (MA.Addr != this) {
512 NodeId MX = MA.Addr->getNext();
513 if (MX == NA.Id) {
514 MA.Addr->setNext(NA.Addr->getNext());
515 // If the member to remove happens to be the last one, update the
516 // LastM indicator.
517 if (Code.LastM == NA.Id)
518 Code.LastM = MA.Id;
519 return;
520 }
521 MA = G.addr<NodeBase*>(MX);
522 }
523 llvm_unreachable("No such member");
524 }
525
526 // Return the list of all members of the code node.
members(const DataFlowGraph & G) const527 NodeList CodeNode::members(const DataFlowGraph &G) const {
528 static auto True = [] (NodeAddr<NodeBase*>) -> bool { return true; };
529 return members_if(True, G);
530 }
531
532 // Return the owner of the given instr node.
getOwner(const DataFlowGraph & G)533 NodeAddr<NodeBase*> InstrNode::getOwner(const DataFlowGraph &G) {
534 NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
535
536 while (NA.Addr != this) {
537 assert(NA.Addr->getType() == NodeAttrs::Code);
538 if (NA.Addr->getKind() == NodeAttrs::Block)
539 return NA;
540 NA = G.addr<NodeBase*>(NA.Addr->getNext());
541 }
542 llvm_unreachable("No owner in circular list");
543 }
544
545 // Add the phi node PA to the given block node.
addPhi(NodeAddr<PhiNode * > PA,const DataFlowGraph & G)546 void BlockNode::addPhi(NodeAddr<PhiNode*> PA, const DataFlowGraph &G) {
547 NodeAddr<NodeBase*> M = getFirstMember(G);
548 if (M.Id == 0) {
549 addMember(PA, G);
550 return;
551 }
552
553 assert(M.Addr->getType() == NodeAttrs::Code);
554 if (M.Addr->getKind() == NodeAttrs::Stmt) {
555 // If the first member of the block is a statement, insert the phi as
556 // the first member.
557 Code.FirstM = PA.Id;
558 PA.Addr->setNext(M.Id);
559 } else {
560 // If the first member is a phi, find the last phi, and append PA to it.
561 assert(M.Addr->getKind() == NodeAttrs::Phi);
562 NodeAddr<NodeBase*> MN = M;
563 do {
564 M = MN;
565 MN = G.addr<NodeBase*>(M.Addr->getNext());
566 assert(MN.Addr->getType() == NodeAttrs::Code);
567 } while (MN.Addr->getKind() == NodeAttrs::Phi);
568
569 // M is the last phi.
570 addMemberAfter(M, PA, G);
571 }
572 }
573
574 // Find the block node corresponding to the machine basic block BB in the
575 // given func node.
findBlock(const MachineBasicBlock * BB,const DataFlowGraph & G) const576 NodeAddr<BlockNode*> FuncNode::findBlock(const MachineBasicBlock *BB,
577 const DataFlowGraph &G) const {
578 auto EqBB = [BB] (NodeAddr<NodeBase*> NA) -> bool {
579 return NodeAddr<BlockNode*>(NA).Addr->getCode() == BB;
580 };
581 NodeList Ms = members_if(EqBB, G);
582 if (!Ms.empty())
583 return Ms[0];
584 return NodeAddr<BlockNode*>();
585 }
586
587 // Get the block node for the entry block in the given function.
getEntryBlock(const DataFlowGraph & G)588 NodeAddr<BlockNode*> FuncNode::getEntryBlock(const DataFlowGraph &G) {
589 MachineBasicBlock *EntryB = &getCode()->front();
590 return findBlock(EntryB, G);
591 }
592
593 // Target operand information.
594 //
595
596 // For a given instruction, check if there are any bits of RR that can remain
597 // unchanged across this def.
isPreserving(const MachineInstr & In,unsigned OpNum) const598 bool TargetOperandInfo::isPreserving(const MachineInstr &In, unsigned OpNum)
599 const {
600 return TII.isPredicated(In);
601 }
602
603 // Check if the definition of RR produces an unspecified value.
isClobbering(const MachineInstr & In,unsigned OpNum) const604 bool TargetOperandInfo::isClobbering(const MachineInstr &In, unsigned OpNum)
605 const {
606 const MachineOperand &Op = In.getOperand(OpNum);
607 if (Op.isRegMask())
608 return true;
609 assert(Op.isReg());
610 if (In.isCall())
611 if (Op.isDef() && Op.isDead())
612 return true;
613 return false;
614 }
615
616 // Check if the given instruction specifically requires
isFixedReg(const MachineInstr & In,unsigned OpNum) const617 bool TargetOperandInfo::isFixedReg(const MachineInstr &In, unsigned OpNum)
618 const {
619 if (In.isCall() || In.isReturn() || In.isInlineAsm())
620 return true;
621 // Check for a tail call.
622 if (In.isBranch())
623 for (const MachineOperand &O : In.operands())
624 if (O.isGlobal() || O.isSymbol())
625 return true;
626
627 const MCInstrDesc &D = In.getDesc();
628 if (!D.getImplicitDefs() && !D.getImplicitUses())
629 return false;
630 const MachineOperand &Op = In.getOperand(OpNum);
631 // If there is a sub-register, treat the operand as non-fixed. Currently,
632 // fixed registers are those that are listed in the descriptor as implicit
633 // uses or defs, and those lists do not allow sub-registers.
634 if (Op.getSubReg() != 0)
635 return false;
636 Register Reg = Op.getReg();
637 const MCPhysReg *ImpR = Op.isDef() ? D.getImplicitDefs()
638 : D.getImplicitUses();
639 if (!ImpR)
640 return false;
641 while (*ImpR)
642 if (*ImpR++ == Reg)
643 return true;
644 return false;
645 }
646
647 //
648 // The data flow graph construction.
649 //
650
DataFlowGraph(MachineFunction & mf,const TargetInstrInfo & tii,const TargetRegisterInfo & tri,const MachineDominatorTree & mdt,const MachineDominanceFrontier & mdf,const TargetOperandInfo & toi)651 DataFlowGraph::DataFlowGraph(MachineFunction &mf, const TargetInstrInfo &tii,
652 const TargetRegisterInfo &tri, const MachineDominatorTree &mdt,
653 const MachineDominanceFrontier &mdf, const TargetOperandInfo &toi)
654 : MF(mf), TII(tii), TRI(tri), PRI(tri, mf), MDT(mdt), MDF(mdf), TOI(toi),
655 LiveIns(PRI) {
656 }
657
658 // The implementation of the definition stack.
659 // Each register reference has its own definition stack. In particular,
660 // for a register references "Reg" and "Reg:subreg" will each have their
661 // own definition stacks.
662
663 // Construct a stack iterator.
Iterator(const DataFlowGraph::DefStack & S,bool Top)664 DataFlowGraph::DefStack::Iterator::Iterator(const DataFlowGraph::DefStack &S,
665 bool Top) : DS(S) {
666 if (!Top) {
667 // Initialize to bottom.
668 Pos = 0;
669 return;
670 }
671 // Initialize to the top, i.e. top-most non-delimiter (or 0, if empty).
672 Pos = DS.Stack.size();
673 while (Pos > 0 && DS.isDelimiter(DS.Stack[Pos-1]))
674 Pos--;
675 }
676
677 // Return the size of the stack, including block delimiters.
size() const678 unsigned DataFlowGraph::DefStack::size() const {
679 unsigned S = 0;
680 for (auto I = top(), E = bottom(); I != E; I.down())
681 S++;
682 return S;
683 }
684
685 // Remove the top entry from the stack. Remove all intervening delimiters
686 // so that after this, the stack is either empty, or the top of the stack
687 // is a non-delimiter.
pop()688 void DataFlowGraph::DefStack::pop() {
689 assert(!empty());
690 unsigned P = nextDown(Stack.size());
691 Stack.resize(P);
692 }
693
694 // Push a delimiter for block node N on the stack.
start_block(NodeId N)695 void DataFlowGraph::DefStack::start_block(NodeId N) {
696 assert(N != 0);
697 Stack.push_back(NodeAddr<DefNode*>(nullptr, N));
698 }
699
700 // Remove all nodes from the top of the stack, until the delimited for
701 // block node N is encountered. Remove the delimiter as well. In effect,
702 // this will remove from the stack all definitions from block N.
clear_block(NodeId N)703 void DataFlowGraph::DefStack::clear_block(NodeId N) {
704 assert(N != 0);
705 unsigned P = Stack.size();
706 while (P > 0) {
707 bool Found = isDelimiter(Stack[P-1], N);
708 P--;
709 if (Found)
710 break;
711 }
712 // This will also remove the delimiter, if found.
713 Stack.resize(P);
714 }
715
716 // Move the stack iterator up by one.
nextUp(unsigned P) const717 unsigned DataFlowGraph::DefStack::nextUp(unsigned P) const {
718 // Get the next valid position after P (skipping all delimiters).
719 // The input position P does not have to point to a non-delimiter.
720 unsigned SS = Stack.size();
721 bool IsDelim;
722 assert(P < SS);
723 do {
724 P++;
725 IsDelim = isDelimiter(Stack[P-1]);
726 } while (P < SS && IsDelim);
727 assert(!IsDelim);
728 return P;
729 }
730
731 // Move the stack iterator down by one.
nextDown(unsigned P) const732 unsigned DataFlowGraph::DefStack::nextDown(unsigned P) const {
733 // Get the preceding valid position before P (skipping all delimiters).
734 // The input position P does not have to point to a non-delimiter.
735 assert(P > 0 && P <= Stack.size());
736 bool IsDelim = isDelimiter(Stack[P-1]);
737 do {
738 if (--P == 0)
739 break;
740 IsDelim = isDelimiter(Stack[P-1]);
741 } while (P > 0 && IsDelim);
742 assert(!IsDelim);
743 return P;
744 }
745
746 // Register information.
747
getLandingPadLiveIns() const748 RegisterSet DataFlowGraph::getLandingPadLiveIns() const {
749 RegisterSet LR;
750 const Function &F = MF.getFunction();
751 const Constant *PF = F.hasPersonalityFn() ? F.getPersonalityFn()
752 : nullptr;
753 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
754 if (RegisterId R = TLI.getExceptionPointerRegister(PF))
755 LR.insert(RegisterRef(R));
756 if (!isFuncletEHPersonality(classifyEHPersonality(PF))) {
757 if (RegisterId R = TLI.getExceptionSelectorRegister(PF))
758 LR.insert(RegisterRef(R));
759 }
760 return LR;
761 }
762
763 // Node management functions.
764
765 // Get the pointer to the node with the id N.
ptr(NodeId N) const766 NodeBase *DataFlowGraph::ptr(NodeId N) const {
767 if (N == 0)
768 return nullptr;
769 return Memory.ptr(N);
770 }
771
772 // Get the id of the node at the address P.
id(const NodeBase * P) const773 NodeId DataFlowGraph::id(const NodeBase *P) const {
774 if (P == nullptr)
775 return 0;
776 return Memory.id(P);
777 }
778
779 // Allocate a new node and set the attributes to Attrs.
newNode(uint16_t Attrs)780 NodeAddr<NodeBase*> DataFlowGraph::newNode(uint16_t Attrs) {
781 NodeAddr<NodeBase*> P = Memory.New();
782 P.Addr->init();
783 P.Addr->setAttrs(Attrs);
784 return P;
785 }
786
787 // Make a copy of the given node B, except for the data-flow links, which
788 // are set to 0.
cloneNode(const NodeAddr<NodeBase * > B)789 NodeAddr<NodeBase*> DataFlowGraph::cloneNode(const NodeAddr<NodeBase*> B) {
790 NodeAddr<NodeBase*> NA = newNode(0);
791 memcpy(NA.Addr, B.Addr, sizeof(NodeBase));
792 // Ref nodes need to have the data-flow links reset.
793 if (NA.Addr->getType() == NodeAttrs::Ref) {
794 NodeAddr<RefNode*> RA = NA;
795 RA.Addr->setReachingDef(0);
796 RA.Addr->setSibling(0);
797 if (NA.Addr->getKind() == NodeAttrs::Def) {
798 NodeAddr<DefNode*> DA = NA;
799 DA.Addr->setReachedDef(0);
800 DA.Addr->setReachedUse(0);
801 }
802 }
803 return NA;
804 }
805
806 // Allocation routines for specific node types/kinds.
807
newUse(NodeAddr<InstrNode * > Owner,MachineOperand & Op,uint16_t Flags)808 NodeAddr<UseNode*> DataFlowGraph::newUse(NodeAddr<InstrNode*> Owner,
809 MachineOperand &Op, uint16_t Flags) {
810 NodeAddr<UseNode*> UA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
811 UA.Addr->setRegRef(&Op, *this);
812 return UA;
813 }
814
newPhiUse(NodeAddr<PhiNode * > Owner,RegisterRef RR,NodeAddr<BlockNode * > PredB,uint16_t Flags)815 NodeAddr<PhiUseNode*> DataFlowGraph::newPhiUse(NodeAddr<PhiNode*> Owner,
816 RegisterRef RR, NodeAddr<BlockNode*> PredB, uint16_t Flags) {
817 NodeAddr<PhiUseNode*> PUA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
818 assert(Flags & NodeAttrs::PhiRef);
819 PUA.Addr->setRegRef(RR, *this);
820 PUA.Addr->setPredecessor(PredB.Id);
821 return PUA;
822 }
823
newDef(NodeAddr<InstrNode * > Owner,MachineOperand & Op,uint16_t Flags)824 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
825 MachineOperand &Op, uint16_t Flags) {
826 NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
827 DA.Addr->setRegRef(&Op, *this);
828 return DA;
829 }
830
newDef(NodeAddr<InstrNode * > Owner,RegisterRef RR,uint16_t Flags)831 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
832 RegisterRef RR, uint16_t Flags) {
833 NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
834 assert(Flags & NodeAttrs::PhiRef);
835 DA.Addr->setRegRef(RR, *this);
836 return DA;
837 }
838
newPhi(NodeAddr<BlockNode * > Owner)839 NodeAddr<PhiNode*> DataFlowGraph::newPhi(NodeAddr<BlockNode*> Owner) {
840 NodeAddr<PhiNode*> PA = newNode(NodeAttrs::Code | NodeAttrs::Phi);
841 Owner.Addr->addPhi(PA, *this);
842 return PA;
843 }
844
newStmt(NodeAddr<BlockNode * > Owner,MachineInstr * MI)845 NodeAddr<StmtNode*> DataFlowGraph::newStmt(NodeAddr<BlockNode*> Owner,
846 MachineInstr *MI) {
847 NodeAddr<StmtNode*> SA = newNode(NodeAttrs::Code | NodeAttrs::Stmt);
848 SA.Addr->setCode(MI);
849 Owner.Addr->addMember(SA, *this);
850 return SA;
851 }
852
newBlock(NodeAddr<FuncNode * > Owner,MachineBasicBlock * BB)853 NodeAddr<BlockNode*> DataFlowGraph::newBlock(NodeAddr<FuncNode*> Owner,
854 MachineBasicBlock *BB) {
855 NodeAddr<BlockNode*> BA = newNode(NodeAttrs::Code | NodeAttrs::Block);
856 BA.Addr->setCode(BB);
857 Owner.Addr->addMember(BA, *this);
858 return BA;
859 }
860
newFunc(MachineFunction * MF)861 NodeAddr<FuncNode*> DataFlowGraph::newFunc(MachineFunction *MF) {
862 NodeAddr<FuncNode*> FA = newNode(NodeAttrs::Code | NodeAttrs::Func);
863 FA.Addr->setCode(MF);
864 return FA;
865 }
866
867 // Build the data flow graph.
build(unsigned Options)868 void DataFlowGraph::build(unsigned Options) {
869 reset();
870 Func = newFunc(&MF);
871
872 if (MF.empty())
873 return;
874
875 for (MachineBasicBlock &B : MF) {
876 NodeAddr<BlockNode*> BA = newBlock(Func, &B);
877 BlockNodes.insert(std::make_pair(&B, BA));
878 for (MachineInstr &I : B) {
879 if (I.isDebugInstr())
880 continue;
881 buildStmt(BA, I);
882 }
883 }
884
885 NodeAddr<BlockNode*> EA = Func.Addr->getEntryBlock(*this);
886 NodeList Blocks = Func.Addr->members(*this);
887
888 // Collect information about block references.
889 RegisterSet AllRefs;
890 for (NodeAddr<BlockNode*> BA : Blocks)
891 for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this))
892 for (NodeAddr<RefNode*> RA : IA.Addr->members(*this))
893 AllRefs.insert(RA.Addr->getRegRef(*this));
894
895 // Collect function live-ins and entry block live-ins.
896 MachineRegisterInfo &MRI = MF.getRegInfo();
897 MachineBasicBlock &EntryB = *EA.Addr->getCode();
898 assert(EntryB.pred_empty() && "Function entry block has predecessors");
899 for (std::pair<unsigned,unsigned> P : MRI.liveins())
900 LiveIns.insert(RegisterRef(P.first));
901 if (MRI.tracksLiveness()) {
902 for (auto I : EntryB.liveins())
903 LiveIns.insert(RegisterRef(I.PhysReg, I.LaneMask));
904 }
905
906 // Add function-entry phi nodes for the live-in registers.
907 //for (std::pair<RegisterId,LaneBitmask> P : LiveIns) {
908 for (auto I = LiveIns.rr_begin(), E = LiveIns.rr_end(); I != E; ++I) {
909 RegisterRef RR = *I;
910 NodeAddr<PhiNode*> PA = newPhi(EA);
911 uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
912 NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
913 PA.Addr->addMember(DA, *this);
914 }
915
916 // Add phis for landing pads.
917 // Landing pads, unlike usual backs blocks, are not entered through
918 // branches in the program, or fall-throughs from other blocks. They
919 // are entered from the exception handling runtime and target's ABI
920 // may define certain registers as defined on entry to such a block.
921 RegisterSet EHRegs = getLandingPadLiveIns();
922 if (!EHRegs.empty()) {
923 for (NodeAddr<BlockNode*> BA : Blocks) {
924 const MachineBasicBlock &B = *BA.Addr->getCode();
925 if (!B.isEHPad())
926 continue;
927
928 // Prepare a list of NodeIds of the block's predecessors.
929 NodeList Preds;
930 for (MachineBasicBlock *PB : B.predecessors())
931 Preds.push_back(findBlock(PB));
932
933 // Build phi nodes for each live-in.
934 for (RegisterRef RR : EHRegs) {
935 NodeAddr<PhiNode*> PA = newPhi(BA);
936 uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
937 // Add def:
938 NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
939 PA.Addr->addMember(DA, *this);
940 // Add uses (no reaching defs for phi uses):
941 for (NodeAddr<BlockNode*> PBA : Preds) {
942 NodeAddr<PhiUseNode*> PUA = newPhiUse(PA, RR, PBA);
943 PA.Addr->addMember(PUA, *this);
944 }
945 }
946 }
947 }
948
949 // Build a map "PhiM" which will contain, for each block, the set
950 // of references that will require phi definitions in that block.
951 BlockRefsMap PhiM;
952 for (NodeAddr<BlockNode*> BA : Blocks)
953 recordDefsForDF(PhiM, BA);
954 for (NodeAddr<BlockNode*> BA : Blocks)
955 buildPhis(PhiM, AllRefs, BA);
956
957 // Link all the refs. This will recursively traverse the dominator tree.
958 DefStackMap DM;
959 linkBlockRefs(DM, EA);
960
961 // Finally, remove all unused phi nodes.
962 if (!(Options & BuildOptions::KeepDeadPhis))
963 removeUnusedPhis();
964 }
965
makeRegRef(unsigned Reg,unsigned Sub) const966 RegisterRef DataFlowGraph::makeRegRef(unsigned Reg, unsigned Sub) const {
967 assert(PhysicalRegisterInfo::isRegMaskId(Reg) ||
968 Register::isPhysicalRegister(Reg));
969 assert(Reg != 0);
970 if (Sub != 0)
971 Reg = TRI.getSubReg(Reg, Sub);
972 return RegisterRef(Reg);
973 }
974
makeRegRef(const MachineOperand & Op) const975 RegisterRef DataFlowGraph::makeRegRef(const MachineOperand &Op) const {
976 assert(Op.isReg() || Op.isRegMask());
977 if (Op.isReg())
978 return makeRegRef(Op.getReg(), Op.getSubReg());
979 return RegisterRef(PRI.getRegMaskId(Op.getRegMask()), LaneBitmask::getAll());
980 }
981
restrictRef(RegisterRef AR,RegisterRef BR) const982 RegisterRef DataFlowGraph::restrictRef(RegisterRef AR, RegisterRef BR) const {
983 if (AR.Reg == BR.Reg) {
984 LaneBitmask M = AR.Mask & BR.Mask;
985 return M.any() ? RegisterRef(AR.Reg, M) : RegisterRef();
986 }
987 #ifndef NDEBUG
988 // RegisterRef NAR = PRI.normalize(AR);
989 // RegisterRef NBR = PRI.normalize(BR);
990 // assert(NAR.Reg != NBR.Reg);
991 #endif
992 // This isn't strictly correct, because the overlap may happen in the
993 // part masked out.
994 if (PRI.alias(AR, BR))
995 return AR;
996 return RegisterRef();
997 }
998
999 // For each stack in the map DefM, push the delimiter for block B on it.
markBlock(NodeId B,DefStackMap & DefM)1000 void DataFlowGraph::markBlock(NodeId B, DefStackMap &DefM) {
1001 // Push block delimiters.
1002 for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
1003 I->second.start_block(B);
1004 }
1005
1006 // Remove all definitions coming from block B from each stack in DefM.
releaseBlock(NodeId B,DefStackMap & DefM)1007 void DataFlowGraph::releaseBlock(NodeId B, DefStackMap &DefM) {
1008 // Pop all defs from this block from the definition stack. Defs that were
1009 // added to the map during the traversal of instructions will not have a
1010 // delimiter, but for those, the whole stack will be emptied.
1011 for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
1012 I->second.clear_block(B);
1013
1014 // Finally, remove empty stacks from the map.
1015 for (auto I = DefM.begin(), E = DefM.end(), NextI = I; I != E; I = NextI) {
1016 NextI = std::next(I);
1017 // This preserves the validity of iterators other than I.
1018 if (I->second.empty())
1019 DefM.erase(I);
1020 }
1021 }
1022
1023 // Push all definitions from the instruction node IA to an appropriate
1024 // stack in DefM.
pushAllDefs(NodeAddr<InstrNode * > IA,DefStackMap & DefM)1025 void DataFlowGraph::pushAllDefs(NodeAddr<InstrNode*> IA, DefStackMap &DefM) {
1026 pushClobbers(IA, DefM);
1027 pushDefs(IA, DefM);
1028 }
1029
1030 // Push all definitions from the instruction node IA to an appropriate
1031 // stack in DefM.
pushClobbers(NodeAddr<InstrNode * > IA,DefStackMap & DefM)1032 void DataFlowGraph::pushClobbers(NodeAddr<InstrNode*> IA, DefStackMap &DefM) {
1033 NodeSet Visited;
1034 std::set<RegisterId> Defined;
1035
1036 // The important objectives of this function are:
1037 // - to be able to handle instructions both while the graph is being
1038 // constructed, and after the graph has been constructed, and
1039 // - maintain proper ordering of definitions on the stack for each
1040 // register reference:
1041 // - if there are two or more related defs in IA (i.e. coming from
1042 // the same machine operand), then only push one def on the stack,
1043 // - if there are multiple unrelated defs of non-overlapping
1044 // subregisters of S, then the stack for S will have both (in an
1045 // unspecified order), but the order does not matter from the data-
1046 // -flow perspective.
1047
1048 for (NodeAddr<DefNode*> DA : IA.Addr->members_if(IsDef, *this)) {
1049 if (Visited.count(DA.Id))
1050 continue;
1051 if (!(DA.Addr->getFlags() & NodeAttrs::Clobbering))
1052 continue;
1053
1054 NodeList Rel = getRelatedRefs(IA, DA);
1055 NodeAddr<DefNode*> PDA = Rel.front();
1056 RegisterRef RR = PDA.Addr->getRegRef(*this);
1057
1058 // Push the definition on the stack for the register and all aliases.
1059 // The def stack traversal in linkNodeUp will check the exact aliasing.
1060 DefM[RR.Reg].push(DA);
1061 Defined.insert(RR.Reg);
1062 for (RegisterId A : PRI.getAliasSet(RR.Reg)) {
1063 // Check that we don't push the same def twice.
1064 assert(A != RR.Reg);
1065 if (!Defined.count(A))
1066 DefM[A].push(DA);
1067 }
1068 // Mark all the related defs as visited.
1069 for (NodeAddr<NodeBase*> T : Rel)
1070 Visited.insert(T.Id);
1071 }
1072 }
1073
1074 // Push all definitions from the instruction node IA to an appropriate
1075 // stack in DefM.
pushDefs(NodeAddr<InstrNode * > IA,DefStackMap & DefM)1076 void DataFlowGraph::pushDefs(NodeAddr<InstrNode*> IA, DefStackMap &DefM) {
1077 NodeSet Visited;
1078 #ifndef NDEBUG
1079 std::set<RegisterId> Defined;
1080 #endif
1081
1082 // The important objectives of this function are:
1083 // - to be able to handle instructions both while the graph is being
1084 // constructed, and after the graph has been constructed, and
1085 // - maintain proper ordering of definitions on the stack for each
1086 // register reference:
1087 // - if there are two or more related defs in IA (i.e. coming from
1088 // the same machine operand), then only push one def on the stack,
1089 // - if there are multiple unrelated defs of non-overlapping
1090 // subregisters of S, then the stack for S will have both (in an
1091 // unspecified order), but the order does not matter from the data-
1092 // -flow perspective.
1093
1094 for (NodeAddr<DefNode*> DA : IA.Addr->members_if(IsDef, *this)) {
1095 if (Visited.count(DA.Id))
1096 continue;
1097 if (DA.Addr->getFlags() & NodeAttrs::Clobbering)
1098 continue;
1099
1100 NodeList Rel = getRelatedRefs(IA, DA);
1101 NodeAddr<DefNode*> PDA = Rel.front();
1102 RegisterRef RR = PDA.Addr->getRegRef(*this);
1103 #ifndef NDEBUG
1104 // Assert if the register is defined in two or more unrelated defs.
1105 // This could happen if there are two or more def operands defining it.
1106 if (!Defined.insert(RR.Reg).second) {
1107 MachineInstr *MI = NodeAddr<StmtNode*>(IA).Addr->getCode();
1108 dbgs() << "Multiple definitions of register: "
1109 << Print<RegisterRef>(RR, *this) << " in\n " << *MI << "in "
1110 << printMBBReference(*MI->getParent()) << '\n';
1111 llvm_unreachable(nullptr);
1112 }
1113 #endif
1114 // Push the definition on the stack for the register and all aliases.
1115 // The def stack traversal in linkNodeUp will check the exact aliasing.
1116 DefM[RR.Reg].push(DA);
1117 for (RegisterId A : PRI.getAliasSet(RR.Reg)) {
1118 // Check that we don't push the same def twice.
1119 assert(A != RR.Reg);
1120 DefM[A].push(DA);
1121 }
1122 // Mark all the related defs as visited.
1123 for (NodeAddr<NodeBase*> T : Rel)
1124 Visited.insert(T.Id);
1125 }
1126 }
1127
1128 // Return the list of all reference nodes related to RA, including RA itself.
1129 // See "getNextRelated" for the meaning of a "related reference".
getRelatedRefs(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1130 NodeList DataFlowGraph::getRelatedRefs(NodeAddr<InstrNode*> IA,
1131 NodeAddr<RefNode*> RA) const {
1132 assert(IA.Id != 0 && RA.Id != 0);
1133
1134 NodeList Refs;
1135 NodeId Start = RA.Id;
1136 do {
1137 Refs.push_back(RA);
1138 RA = getNextRelated(IA, RA);
1139 } while (RA.Id != 0 && RA.Id != Start);
1140 return Refs;
1141 }
1142
1143 // Clear all information in the graph.
reset()1144 void DataFlowGraph::reset() {
1145 Memory.clear();
1146 BlockNodes.clear();
1147 Func = NodeAddr<FuncNode*>();
1148 }
1149
1150 // Return the next reference node in the instruction node IA that is related
1151 // to RA. Conceptually, two reference nodes are related if they refer to the
1152 // same instance of a register access, but differ in flags or other minor
1153 // characteristics. Specific examples of related nodes are shadow reference
1154 // nodes.
1155 // Return the equivalent of nullptr if there are no more related references.
getNextRelated(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1156 NodeAddr<RefNode*> DataFlowGraph::getNextRelated(NodeAddr<InstrNode*> IA,
1157 NodeAddr<RefNode*> RA) const {
1158 assert(IA.Id != 0 && RA.Id != 0);
1159
1160 auto Related = [this,RA](NodeAddr<RefNode*> TA) -> bool {
1161 if (TA.Addr->getKind() != RA.Addr->getKind())
1162 return false;
1163 if (TA.Addr->getRegRef(*this) != RA.Addr->getRegRef(*this))
1164 return false;
1165 return true;
1166 };
1167 auto RelatedStmt = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1168 return Related(TA) &&
1169 &RA.Addr->getOp() == &TA.Addr->getOp();
1170 };
1171 auto RelatedPhi = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1172 if (!Related(TA))
1173 return false;
1174 if (TA.Addr->getKind() != NodeAttrs::Use)
1175 return true;
1176 // For phi uses, compare predecessor blocks.
1177 const NodeAddr<const PhiUseNode*> TUA = TA;
1178 const NodeAddr<const PhiUseNode*> RUA = RA;
1179 return TUA.Addr->getPredecessor() == RUA.Addr->getPredecessor();
1180 };
1181
1182 RegisterRef RR = RA.Addr->getRegRef(*this);
1183 if (IA.Addr->getKind() == NodeAttrs::Stmt)
1184 return RA.Addr->getNextRef(RR, RelatedStmt, true, *this);
1185 return RA.Addr->getNextRef(RR, RelatedPhi, true, *this);
1186 }
1187
1188 // Find the next node related to RA in IA that satisfies condition P.
1189 // If such a node was found, return a pair where the second element is the
1190 // located node. If such a node does not exist, return a pair where the
1191 // first element is the element after which such a node should be inserted,
1192 // and the second element is a null-address.
1193 template <typename Predicate>
1194 std::pair<NodeAddr<RefNode*>,NodeAddr<RefNode*>>
locateNextRef(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA,Predicate P) const1195 DataFlowGraph::locateNextRef(NodeAddr<InstrNode*> IA, NodeAddr<RefNode*> RA,
1196 Predicate P) const {
1197 assert(IA.Id != 0 && RA.Id != 0);
1198
1199 NodeAddr<RefNode*> NA;
1200 NodeId Start = RA.Id;
1201 while (true) {
1202 NA = getNextRelated(IA, RA);
1203 if (NA.Id == 0 || NA.Id == Start)
1204 break;
1205 if (P(NA))
1206 break;
1207 RA = NA;
1208 }
1209
1210 if (NA.Id != 0 && NA.Id != Start)
1211 return std::make_pair(RA, NA);
1212 return std::make_pair(RA, NodeAddr<RefNode*>());
1213 }
1214
1215 // Get the next shadow node in IA corresponding to RA, and optionally create
1216 // such a node if it does not exist.
getNextShadow(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA,bool Create)1217 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1218 NodeAddr<RefNode*> RA, bool Create) {
1219 assert(IA.Id != 0 && RA.Id != 0);
1220
1221 uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1222 auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1223 return TA.Addr->getFlags() == Flags;
1224 };
1225 auto Loc = locateNextRef(IA, RA, IsShadow);
1226 if (Loc.second.Id != 0 || !Create)
1227 return Loc.second;
1228
1229 // Create a copy of RA and mark is as shadow.
1230 NodeAddr<RefNode*> NA = cloneNode(RA);
1231 NA.Addr->setFlags(Flags | NodeAttrs::Shadow);
1232 IA.Addr->addMemberAfter(Loc.first, NA, *this);
1233 return NA;
1234 }
1235
1236 // Get the next shadow node in IA corresponding to RA. Return null-address
1237 // if such a node does not exist.
getNextShadow(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1238 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1239 NodeAddr<RefNode*> RA) const {
1240 assert(IA.Id != 0 && RA.Id != 0);
1241 uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1242 auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1243 return TA.Addr->getFlags() == Flags;
1244 };
1245 return locateNextRef(IA, RA, IsShadow).second;
1246 }
1247
1248 // Create a new statement node in the block node BA that corresponds to
1249 // the machine instruction MI.
buildStmt(NodeAddr<BlockNode * > BA,MachineInstr & In)1250 void DataFlowGraph::buildStmt(NodeAddr<BlockNode*> BA, MachineInstr &In) {
1251 NodeAddr<StmtNode*> SA = newStmt(BA, &In);
1252
1253 auto isCall = [] (const MachineInstr &In) -> bool {
1254 if (In.isCall())
1255 return true;
1256 // Is tail call?
1257 if (In.isBranch()) {
1258 for (const MachineOperand &Op : In.operands())
1259 if (Op.isGlobal() || Op.isSymbol())
1260 return true;
1261 // Assume indirect branches are calls. This is for the purpose of
1262 // keeping implicit operands, and so it won't hurt on intra-function
1263 // indirect branches.
1264 if (In.isIndirectBranch())
1265 return true;
1266 }
1267 return false;
1268 };
1269
1270 auto isDefUndef = [this] (const MachineInstr &In, RegisterRef DR) -> bool {
1271 // This instruction defines DR. Check if there is a use operand that
1272 // would make DR live on entry to the instruction.
1273 for (const MachineOperand &Op : In.operands()) {
1274 if (!Op.isReg() || Op.getReg() == 0 || !Op.isUse() || Op.isUndef())
1275 continue;
1276 RegisterRef UR = makeRegRef(Op);
1277 if (PRI.alias(DR, UR))
1278 return false;
1279 }
1280 return true;
1281 };
1282
1283 bool IsCall = isCall(In);
1284 unsigned NumOps = In.getNumOperands();
1285
1286 // Avoid duplicate implicit defs. This will not detect cases of implicit
1287 // defs that define registers that overlap, but it is not clear how to
1288 // interpret that in the absence of explicit defs. Overlapping explicit
1289 // defs are likely illegal already.
1290 BitVector DoneDefs(TRI.getNumRegs());
1291 // Process explicit defs first.
1292 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1293 MachineOperand &Op = In.getOperand(OpN);
1294 if (!Op.isReg() || !Op.isDef() || Op.isImplicit())
1295 continue;
1296 Register R = Op.getReg();
1297 if (!R || !Register::isPhysicalRegister(R))
1298 continue;
1299 uint16_t Flags = NodeAttrs::None;
1300 if (TOI.isPreserving(In, OpN)) {
1301 Flags |= NodeAttrs::Preserving;
1302 // If the def is preserving, check if it is also undefined.
1303 if (isDefUndef(In, makeRegRef(Op)))
1304 Flags |= NodeAttrs::Undef;
1305 }
1306 if (TOI.isClobbering(In, OpN))
1307 Flags |= NodeAttrs::Clobbering;
1308 if (TOI.isFixedReg(In, OpN))
1309 Flags |= NodeAttrs::Fixed;
1310 if (IsCall && Op.isDead())
1311 Flags |= NodeAttrs::Dead;
1312 NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1313 SA.Addr->addMember(DA, *this);
1314 assert(!DoneDefs.test(R));
1315 DoneDefs.set(R);
1316 }
1317
1318 // Process reg-masks (as clobbers).
1319 BitVector DoneClobbers(TRI.getNumRegs());
1320 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1321 MachineOperand &Op = In.getOperand(OpN);
1322 if (!Op.isRegMask())
1323 continue;
1324 uint16_t Flags = NodeAttrs::Clobbering | NodeAttrs::Fixed |
1325 NodeAttrs::Dead;
1326 NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1327 SA.Addr->addMember(DA, *this);
1328 // Record all clobbered registers in DoneDefs.
1329 const uint32_t *RM = Op.getRegMask();
1330 for (unsigned i = 1, e = TRI.getNumRegs(); i != e; ++i)
1331 if (!(RM[i/32] & (1u << (i%32))))
1332 DoneClobbers.set(i);
1333 }
1334
1335 // Process implicit defs, skipping those that have already been added
1336 // as explicit.
1337 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1338 MachineOperand &Op = In.getOperand(OpN);
1339 if (!Op.isReg() || !Op.isDef() || !Op.isImplicit())
1340 continue;
1341 Register R = Op.getReg();
1342 if (!R || !Register::isPhysicalRegister(R) || DoneDefs.test(R))
1343 continue;
1344 RegisterRef RR = makeRegRef(Op);
1345 uint16_t Flags = NodeAttrs::None;
1346 if (TOI.isPreserving(In, OpN)) {
1347 Flags |= NodeAttrs::Preserving;
1348 // If the def is preserving, check if it is also undefined.
1349 if (isDefUndef(In, RR))
1350 Flags |= NodeAttrs::Undef;
1351 }
1352 if (TOI.isClobbering(In, OpN))
1353 Flags |= NodeAttrs::Clobbering;
1354 if (TOI.isFixedReg(In, OpN))
1355 Flags |= NodeAttrs::Fixed;
1356 if (IsCall && Op.isDead()) {
1357 if (DoneClobbers.test(R))
1358 continue;
1359 Flags |= NodeAttrs::Dead;
1360 }
1361 NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1362 SA.Addr->addMember(DA, *this);
1363 DoneDefs.set(R);
1364 }
1365
1366 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1367 MachineOperand &Op = In.getOperand(OpN);
1368 if (!Op.isReg() || !Op.isUse())
1369 continue;
1370 Register R = Op.getReg();
1371 if (!R || !Register::isPhysicalRegister(R))
1372 continue;
1373 uint16_t Flags = NodeAttrs::None;
1374 if (Op.isUndef())
1375 Flags |= NodeAttrs::Undef;
1376 if (TOI.isFixedReg(In, OpN))
1377 Flags |= NodeAttrs::Fixed;
1378 NodeAddr<UseNode*> UA = newUse(SA, Op, Flags);
1379 SA.Addr->addMember(UA, *this);
1380 }
1381 }
1382
1383 // Scan all defs in the block node BA and record in PhiM the locations of
1384 // phi nodes corresponding to these defs.
recordDefsForDF(BlockRefsMap & PhiM,NodeAddr<BlockNode * > BA)1385 void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM,
1386 NodeAddr<BlockNode*> BA) {
1387 // Check all defs from block BA and record them in each block in BA's
1388 // iterated dominance frontier. This information will later be used to
1389 // create phi nodes.
1390 MachineBasicBlock *BB = BA.Addr->getCode();
1391 assert(BB);
1392 auto DFLoc = MDF.find(BB);
1393 if (DFLoc == MDF.end() || DFLoc->second.empty())
1394 return;
1395
1396 // Traverse all instructions in the block and collect the set of all
1397 // defined references. For each reference there will be a phi created
1398 // in the block's iterated dominance frontier.
1399 // This is done to make sure that each defined reference gets only one
1400 // phi node, even if it is defined multiple times.
1401 RegisterSet Defs;
1402 for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this))
1403 for (NodeAddr<RefNode*> RA : IA.Addr->members_if(IsDef, *this))
1404 Defs.insert(RA.Addr->getRegRef(*this));
1405
1406 // Calculate the iterated dominance frontier of BB.
1407 const MachineDominanceFrontier::DomSetType &DF = DFLoc->second;
1408 SetVector<MachineBasicBlock*> IDF(DF.begin(), DF.end());
1409 for (unsigned i = 0; i < IDF.size(); ++i) {
1410 auto F = MDF.find(IDF[i]);
1411 if (F != MDF.end())
1412 IDF.insert(F->second.begin(), F->second.end());
1413 }
1414
1415 // Finally, add the set of defs to each block in the iterated dominance
1416 // frontier.
1417 for (auto DB : IDF) {
1418 NodeAddr<BlockNode*> DBA = findBlock(DB);
1419 PhiM[DBA.Id].insert(Defs.begin(), Defs.end());
1420 }
1421 }
1422
1423 // Given the locations of phi nodes in the map PhiM, create the phi nodes
1424 // that are located in the block node BA.
buildPhis(BlockRefsMap & PhiM,RegisterSet & AllRefs,NodeAddr<BlockNode * > BA)1425 void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs,
1426 NodeAddr<BlockNode*> BA) {
1427 // Check if this blocks has any DF defs, i.e. if there are any defs
1428 // that this block is in the iterated dominance frontier of.
1429 auto HasDF = PhiM.find(BA.Id);
1430 if (HasDF == PhiM.end() || HasDF->second.empty())
1431 return;
1432
1433 // First, remove all R in Refs in such that there exists T in Refs
1434 // such that T covers R. In other words, only leave those refs that
1435 // are not covered by another ref (i.e. maximal with respect to covering).
1436
1437 auto MaxCoverIn = [this] (RegisterRef RR, RegisterSet &RRs) -> RegisterRef {
1438 for (RegisterRef I : RRs)
1439 if (I != RR && RegisterAggr::isCoverOf(I, RR, PRI))
1440 RR = I;
1441 return RR;
1442 };
1443
1444 RegisterSet MaxDF;
1445 for (RegisterRef I : HasDF->second)
1446 MaxDF.insert(MaxCoverIn(I, HasDF->second));
1447
1448 std::vector<RegisterRef> MaxRefs;
1449 for (RegisterRef I : MaxDF)
1450 MaxRefs.push_back(MaxCoverIn(I, AllRefs));
1451
1452 // Now, for each R in MaxRefs, get the alias closure of R. If the closure
1453 // only has R in it, create a phi a def for R. Otherwise, create a phi,
1454 // and add a def for each S in the closure.
1455
1456 // Sort the refs so that the phis will be created in a deterministic order.
1457 llvm::sort(MaxRefs);
1458 // Remove duplicates.
1459 auto NewEnd = std::unique(MaxRefs.begin(), MaxRefs.end());
1460 MaxRefs.erase(NewEnd, MaxRefs.end());
1461
1462 auto Aliased = [this,&MaxRefs](RegisterRef RR,
1463 std::vector<unsigned> &Closure) -> bool {
1464 for (unsigned I : Closure)
1465 if (PRI.alias(RR, MaxRefs[I]))
1466 return true;
1467 return false;
1468 };
1469
1470 // Prepare a list of NodeIds of the block's predecessors.
1471 NodeList Preds;
1472 const MachineBasicBlock *MBB = BA.Addr->getCode();
1473 for (MachineBasicBlock *PB : MBB->predecessors())
1474 Preds.push_back(findBlock(PB));
1475
1476 while (!MaxRefs.empty()) {
1477 // Put the first element in the closure, and then add all subsequent
1478 // elements from MaxRefs to it, if they alias at least one element
1479 // already in the closure.
1480 // ClosureIdx: vector of indices in MaxRefs of members of the closure.
1481 std::vector<unsigned> ClosureIdx = { 0 };
1482 for (unsigned i = 1; i != MaxRefs.size(); ++i)
1483 if (Aliased(MaxRefs[i], ClosureIdx))
1484 ClosureIdx.push_back(i);
1485
1486 // Build a phi for the closure.
1487 unsigned CS = ClosureIdx.size();
1488 NodeAddr<PhiNode*> PA = newPhi(BA);
1489
1490 // Add defs.
1491 for (unsigned X = 0; X != CS; ++X) {
1492 RegisterRef RR = MaxRefs[ClosureIdx[X]];
1493 uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
1494 NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
1495 PA.Addr->addMember(DA, *this);
1496 }
1497 // Add phi uses.
1498 for (NodeAddr<BlockNode*> PBA : Preds) {
1499 for (unsigned X = 0; X != CS; ++X) {
1500 RegisterRef RR = MaxRefs[ClosureIdx[X]];
1501 NodeAddr<PhiUseNode*> PUA = newPhiUse(PA, RR, PBA);
1502 PA.Addr->addMember(PUA, *this);
1503 }
1504 }
1505
1506 // Erase from MaxRefs all elements in the closure.
1507 auto Begin = MaxRefs.begin();
1508 for (unsigned i = ClosureIdx.size(); i != 0; --i)
1509 MaxRefs.erase(Begin + ClosureIdx[i-1]);
1510 }
1511 }
1512
1513 // Remove any unneeded phi nodes that were created during the build process.
removeUnusedPhis()1514 void DataFlowGraph::removeUnusedPhis() {
1515 // This will remove unused phis, i.e. phis where each def does not reach
1516 // any uses or other defs. This will not detect or remove circular phi
1517 // chains that are otherwise dead. Unused/dead phis are created during
1518 // the build process and this function is intended to remove these cases
1519 // that are easily determinable to be unnecessary.
1520
1521 SetVector<NodeId> PhiQ;
1522 for (NodeAddr<BlockNode*> BA : Func.Addr->members(*this)) {
1523 for (auto P : BA.Addr->members_if(IsPhi, *this))
1524 PhiQ.insert(P.Id);
1525 }
1526
1527 static auto HasUsedDef = [](NodeList &Ms) -> bool {
1528 for (NodeAddr<NodeBase*> M : Ms) {
1529 if (M.Addr->getKind() != NodeAttrs::Def)
1530 continue;
1531 NodeAddr<DefNode*> DA = M;
1532 if (DA.Addr->getReachedDef() != 0 || DA.Addr->getReachedUse() != 0)
1533 return true;
1534 }
1535 return false;
1536 };
1537
1538 // Any phi, if it is removed, may affect other phis (make them dead).
1539 // For each removed phi, collect the potentially affected phis and add
1540 // them back to the queue.
1541 while (!PhiQ.empty()) {
1542 auto PA = addr<PhiNode*>(PhiQ[0]);
1543 PhiQ.remove(PA.Id);
1544 NodeList Refs = PA.Addr->members(*this);
1545 if (HasUsedDef(Refs))
1546 continue;
1547 for (NodeAddr<RefNode*> RA : Refs) {
1548 if (NodeId RD = RA.Addr->getReachingDef()) {
1549 auto RDA = addr<DefNode*>(RD);
1550 NodeAddr<InstrNode*> OA = RDA.Addr->getOwner(*this);
1551 if (IsPhi(OA))
1552 PhiQ.insert(OA.Id);
1553 }
1554 if (RA.Addr->isDef())
1555 unlinkDef(RA, true);
1556 else
1557 unlinkUse(RA, true);
1558 }
1559 NodeAddr<BlockNode*> BA = PA.Addr->getOwner(*this);
1560 BA.Addr->removeMember(PA, *this);
1561 }
1562 }
1563
1564 // For a given reference node TA in an instruction node IA, connect the
1565 // reaching def of TA to the appropriate def node. Create any shadow nodes
1566 // as appropriate.
1567 template <typename T>
linkRefUp(NodeAddr<InstrNode * > IA,NodeAddr<T> TA,DefStack & DS)1568 void DataFlowGraph::linkRefUp(NodeAddr<InstrNode*> IA, NodeAddr<T> TA,
1569 DefStack &DS) {
1570 if (DS.empty())
1571 return;
1572 RegisterRef RR = TA.Addr->getRegRef(*this);
1573 NodeAddr<T> TAP;
1574
1575 // References from the def stack that have been examined so far.
1576 RegisterAggr Defs(PRI);
1577
1578 for (auto I = DS.top(), E = DS.bottom(); I != E; I.down()) {
1579 RegisterRef QR = I->Addr->getRegRef(*this);
1580
1581 // Skip all defs that are aliased to any of the defs that we have already
1582 // seen. If this completes a cover of RR, stop the stack traversal.
1583 bool Alias = Defs.hasAliasOf(QR);
1584 bool Cover = Defs.insert(QR).hasCoverOf(RR);
1585 if (Alias) {
1586 if (Cover)
1587 break;
1588 continue;
1589 }
1590
1591 // The reaching def.
1592 NodeAddr<DefNode*> RDA = *I;
1593
1594 // Pick the reached node.
1595 if (TAP.Id == 0) {
1596 TAP = TA;
1597 } else {
1598 // Mark the existing ref as "shadow" and create a new shadow.
1599 TAP.Addr->setFlags(TAP.Addr->getFlags() | NodeAttrs::Shadow);
1600 TAP = getNextShadow(IA, TAP, true);
1601 }
1602
1603 // Create the link.
1604 TAP.Addr->linkToDef(TAP.Id, RDA);
1605
1606 if (Cover)
1607 break;
1608 }
1609 }
1610
1611 // Create data-flow links for all reference nodes in the statement node SA.
1612 template <typename Predicate>
linkStmtRefs(DefStackMap & DefM,NodeAddr<StmtNode * > SA,Predicate P)1613 void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, NodeAddr<StmtNode*> SA,
1614 Predicate P) {
1615 #ifndef NDEBUG
1616 RegisterSet Defs;
1617 #endif
1618
1619 // Link all nodes (upwards in the data-flow) with their reaching defs.
1620 for (NodeAddr<RefNode*> RA : SA.Addr->members_if(P, *this)) {
1621 uint16_t Kind = RA.Addr->getKind();
1622 assert(Kind == NodeAttrs::Def || Kind == NodeAttrs::Use);
1623 RegisterRef RR = RA.Addr->getRegRef(*this);
1624 #ifndef NDEBUG
1625 // Do not expect multiple defs of the same reference.
1626 assert(Kind != NodeAttrs::Def || !Defs.count(RR));
1627 Defs.insert(RR);
1628 #endif
1629
1630 auto F = DefM.find(RR.Reg);
1631 if (F == DefM.end())
1632 continue;
1633 DefStack &DS = F->second;
1634 if (Kind == NodeAttrs::Use)
1635 linkRefUp<UseNode*>(SA, RA, DS);
1636 else if (Kind == NodeAttrs::Def)
1637 linkRefUp<DefNode*>(SA, RA, DS);
1638 else
1639 llvm_unreachable("Unexpected node in instruction");
1640 }
1641 }
1642
1643 // Create data-flow links for all instructions in the block node BA. This
1644 // will include updating any phi nodes in BA.
linkBlockRefs(DefStackMap & DefM,NodeAddr<BlockNode * > BA)1645 void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, NodeAddr<BlockNode*> BA) {
1646 // Push block delimiters.
1647 markBlock(BA.Id, DefM);
1648
1649 auto IsClobber = [] (NodeAddr<RefNode*> RA) -> bool {
1650 return IsDef(RA) && (RA.Addr->getFlags() & NodeAttrs::Clobbering);
1651 };
1652 auto IsNoClobber = [] (NodeAddr<RefNode*> RA) -> bool {
1653 return IsDef(RA) && !(RA.Addr->getFlags() & NodeAttrs::Clobbering);
1654 };
1655
1656 assert(BA.Addr && "block node address is needed to create a data-flow link");
1657 // For each non-phi instruction in the block, link all the defs and uses
1658 // to their reaching defs. For any member of the block (including phis),
1659 // push the defs on the corresponding stacks.
1660 for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this)) {
1661 // Ignore phi nodes here. They will be linked part by part from the
1662 // predecessors.
1663 if (IA.Addr->getKind() == NodeAttrs::Stmt) {
1664 linkStmtRefs(DefM, IA, IsUse);
1665 linkStmtRefs(DefM, IA, IsClobber);
1666 }
1667
1668 // Push the definitions on the stack.
1669 pushClobbers(IA, DefM);
1670
1671 if (IA.Addr->getKind() == NodeAttrs::Stmt)
1672 linkStmtRefs(DefM, IA, IsNoClobber);
1673
1674 pushDefs(IA, DefM);
1675 }
1676
1677 // Recursively process all children in the dominator tree.
1678 MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode());
1679 for (auto I : *N) {
1680 MachineBasicBlock *SB = I->getBlock();
1681 NodeAddr<BlockNode*> SBA = findBlock(SB);
1682 linkBlockRefs(DefM, SBA);
1683 }
1684
1685 // Link the phi uses from the successor blocks.
1686 auto IsUseForBA = [BA](NodeAddr<NodeBase*> NA) -> bool {
1687 if (NA.Addr->getKind() != NodeAttrs::Use)
1688 return false;
1689 assert(NA.Addr->getFlags() & NodeAttrs::PhiRef);
1690 NodeAddr<PhiUseNode*> PUA = NA;
1691 return PUA.Addr->getPredecessor() == BA.Id;
1692 };
1693
1694 RegisterSet EHLiveIns = getLandingPadLiveIns();
1695 MachineBasicBlock *MBB = BA.Addr->getCode();
1696
1697 for (MachineBasicBlock *SB : MBB->successors()) {
1698 bool IsEHPad = SB->isEHPad();
1699 NodeAddr<BlockNode*> SBA = findBlock(SB);
1700 for (NodeAddr<InstrNode*> IA : SBA.Addr->members_if(IsPhi, *this)) {
1701 // Do not link phi uses for landing pad live-ins.
1702 if (IsEHPad) {
1703 // Find what register this phi is for.
1704 NodeAddr<RefNode*> RA = IA.Addr->getFirstMember(*this);
1705 assert(RA.Id != 0);
1706 if (EHLiveIns.count(RA.Addr->getRegRef(*this)))
1707 continue;
1708 }
1709 // Go over each phi use associated with MBB, and link it.
1710 for (auto U : IA.Addr->members_if(IsUseForBA, *this)) {
1711 NodeAddr<PhiUseNode*> PUA = U;
1712 RegisterRef RR = PUA.Addr->getRegRef(*this);
1713 linkRefUp<UseNode*>(IA, PUA, DefM[RR.Reg]);
1714 }
1715 }
1716 }
1717
1718 // Pop all defs from this block from the definition stacks.
1719 releaseBlock(BA.Id, DefM);
1720 }
1721
1722 // Remove the use node UA from any data-flow and structural links.
unlinkUseDF(NodeAddr<UseNode * > UA)1723 void DataFlowGraph::unlinkUseDF(NodeAddr<UseNode*> UA) {
1724 NodeId RD = UA.Addr->getReachingDef();
1725 NodeId Sib = UA.Addr->getSibling();
1726
1727 if (RD == 0) {
1728 assert(Sib == 0);
1729 return;
1730 }
1731
1732 auto RDA = addr<DefNode*>(RD);
1733 auto TA = addr<UseNode*>(RDA.Addr->getReachedUse());
1734 if (TA.Id == UA.Id) {
1735 RDA.Addr->setReachedUse(Sib);
1736 return;
1737 }
1738
1739 while (TA.Id != 0) {
1740 NodeId S = TA.Addr->getSibling();
1741 if (S == UA.Id) {
1742 TA.Addr->setSibling(UA.Addr->getSibling());
1743 return;
1744 }
1745 TA = addr<UseNode*>(S);
1746 }
1747 }
1748
1749 // Remove the def node DA from any data-flow and structural links.
unlinkDefDF(NodeAddr<DefNode * > DA)1750 void DataFlowGraph::unlinkDefDF(NodeAddr<DefNode*> DA) {
1751 //
1752 // RD
1753 // | reached
1754 // | def
1755 // :
1756 // .
1757 // +----+
1758 // ... -- | DA | -- ... -- 0 : sibling chain of DA
1759 // +----+
1760 // | | reached
1761 // | : def
1762 // | .
1763 // | ... : Siblings (defs)
1764 // |
1765 // : reached
1766 // . use
1767 // ... : sibling chain of reached uses
1768
1769 NodeId RD = DA.Addr->getReachingDef();
1770
1771 // Visit all siblings of the reached def and reset their reaching defs.
1772 // Also, defs reached by DA are now "promoted" to being reached by RD,
1773 // so all of them will need to be spliced into the sibling chain where
1774 // DA belongs.
1775 auto getAllNodes = [this] (NodeId N) -> NodeList {
1776 NodeList Res;
1777 while (N) {
1778 auto RA = addr<RefNode*>(N);
1779 // Keep the nodes in the exact sibling order.
1780 Res.push_back(RA);
1781 N = RA.Addr->getSibling();
1782 }
1783 return Res;
1784 };
1785 NodeList ReachedDefs = getAllNodes(DA.Addr->getReachedDef());
1786 NodeList ReachedUses = getAllNodes(DA.Addr->getReachedUse());
1787
1788 if (RD == 0) {
1789 for (NodeAddr<RefNode*> I : ReachedDefs)
1790 I.Addr->setSibling(0);
1791 for (NodeAddr<RefNode*> I : ReachedUses)
1792 I.Addr->setSibling(0);
1793 }
1794 for (NodeAddr<DefNode*> I : ReachedDefs)
1795 I.Addr->setReachingDef(RD);
1796 for (NodeAddr<UseNode*> I : ReachedUses)
1797 I.Addr->setReachingDef(RD);
1798
1799 NodeId Sib = DA.Addr->getSibling();
1800 if (RD == 0) {
1801 assert(Sib == 0);
1802 return;
1803 }
1804
1805 // Update the reaching def node and remove DA from the sibling list.
1806 auto RDA = addr<DefNode*>(RD);
1807 auto TA = addr<DefNode*>(RDA.Addr->getReachedDef());
1808 if (TA.Id == DA.Id) {
1809 // If DA is the first reached def, just update the RD's reached def
1810 // to the DA's sibling.
1811 RDA.Addr->setReachedDef(Sib);
1812 } else {
1813 // Otherwise, traverse the sibling list of the reached defs and remove
1814 // DA from it.
1815 while (TA.Id != 0) {
1816 NodeId S = TA.Addr->getSibling();
1817 if (S == DA.Id) {
1818 TA.Addr->setSibling(Sib);
1819 break;
1820 }
1821 TA = addr<DefNode*>(S);
1822 }
1823 }
1824
1825 // Splice the DA's reached defs into the RDA's reached def chain.
1826 if (!ReachedDefs.empty()) {
1827 auto Last = NodeAddr<DefNode*>(ReachedDefs.back());
1828 Last.Addr->setSibling(RDA.Addr->getReachedDef());
1829 RDA.Addr->setReachedDef(ReachedDefs.front().Id);
1830 }
1831 // Splice the DA's reached uses into the RDA's reached use chain.
1832 if (!ReachedUses.empty()) {
1833 auto Last = NodeAddr<UseNode*>(ReachedUses.back());
1834 Last.Addr->setSibling(RDA.Addr->getReachedUse());
1835 RDA.Addr->setReachedUse(ReachedUses.front().Id);
1836 }
1837 }
1838