1 //===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Analysis/CGSCCPassManager.h"
10 #include "llvm/ADT/ArrayRef.h"
11 #include "llvm/ADT/Optional.h"
12 #include "llvm/ADT/STLExtras.h"
13 #include "llvm/ADT/SetVector.h"
14 #include "llvm/ADT/SmallPtrSet.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/iterator_range.h"
17 #include "llvm/Analysis/LazyCallGraph.h"
18 #include "llvm/IR/Constant.h"
19 #include "llvm/IR/InstIterator.h"
20 #include "llvm/IR/Instruction.h"
21 #include "llvm/IR/PassManager.h"
22 #include "llvm/IR/PassManagerImpl.h"
23 #include "llvm/Support/Casting.h"
24 #include "llvm/Support/Debug.h"
25 #include "llvm/Support/raw_ostream.h"
26 #include "llvm/Support/TimeProfiler.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <iterator>
30
31 #define DEBUG_TYPE "cgscc"
32
33 using namespace llvm;
34
35 // Explicit template instantiations and specialization definitions for core
36 // template typedefs.
37 namespace llvm {
38
39 // Explicit instantiations for the core proxy templates.
40 template class AllAnalysesOn<LazyCallGraph::SCC>;
41 template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>;
42 template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager,
43 LazyCallGraph &, CGSCCUpdateResult &>;
44 template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>;
45 template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
46 LazyCallGraph::SCC, LazyCallGraph &>;
47 template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>;
48
49 /// Explicitly specialize the pass manager run method to handle call graph
50 /// updates.
51 template <>
52 PreservedAnalyses
53 PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
run(LazyCallGraph::SCC & InitialC,CGSCCAnalysisManager & AM,LazyCallGraph & G,CGSCCUpdateResult & UR)54 CGSCCUpdateResult &>::run(LazyCallGraph::SCC &InitialC,
55 CGSCCAnalysisManager &AM,
56 LazyCallGraph &G, CGSCCUpdateResult &UR) {
57 // Request PassInstrumentation from analysis manager, will use it to run
58 // instrumenting callbacks for the passes later.
59 PassInstrumentation PI =
60 AM.getResult<PassInstrumentationAnalysis>(InitialC, G);
61
62 PreservedAnalyses PA = PreservedAnalyses::all();
63
64 if (DebugLogging)
65 dbgs() << "Starting CGSCC pass manager run.\n";
66
67 // The SCC may be refined while we are running passes over it, so set up
68 // a pointer that we can update.
69 LazyCallGraph::SCC *C = &InitialC;
70
71 // Get Function analysis manager from its proxy.
72 FunctionAnalysisManager &FAM =
73 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*C)->getManager();
74
75 for (auto &Pass : Passes) {
76 // Check the PassInstrumentation's BeforePass callbacks before running the
77 // pass, skip its execution completely if asked to (callback returns false).
78 if (!PI.runBeforePass(*Pass, *C))
79 continue;
80
81 if (DebugLogging)
82 dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n";
83
84 PreservedAnalyses PassPA;
85 {
86 TimeTraceScope TimeScope(Pass->name());
87 PassPA = Pass->run(*C, AM, G, UR);
88 }
89
90 if (UR.InvalidatedSCCs.count(C))
91 PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass);
92 else
93 PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C);
94
95 // Update the SCC if necessary.
96 C = UR.UpdatedC ? UR.UpdatedC : C;
97 if (UR.UpdatedC) {
98 // If C is updated, also create a proxy and update FAM inside the result.
99 auto *ResultFAMCP =
100 &AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G);
101 ResultFAMCP->updateFAM(FAM);
102 }
103
104 // If the CGSCC pass wasn't able to provide a valid updated SCC, the
105 // current SCC may simply need to be skipped if invalid.
106 if (UR.InvalidatedSCCs.count(C)) {
107 LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
108 break;
109 }
110 // Check that we didn't miss any update scenario.
111 assert(C->begin() != C->end() && "Cannot have an empty SCC!");
112
113 // Update the analysis manager as each pass runs and potentially
114 // invalidates analyses.
115 AM.invalidate(*C, PassPA);
116
117 // Finally, we intersect the final preserved analyses to compute the
118 // aggregate preserved set for this pass manager.
119 PA.intersect(std::move(PassPA));
120
121 // FIXME: Historically, the pass managers all called the LLVM context's
122 // yield function here. We don't have a generic way to acquire the
123 // context and it isn't yet clear what the right pattern is for yielding
124 // in the new pass manager so it is currently omitted.
125 // ...getContext().yield();
126 }
127
128 // Before we mark all of *this* SCC's analyses as preserved below, intersect
129 // this with the cross-SCC preserved analysis set. This is used to allow
130 // CGSCC passes to mutate ancestor SCCs and still trigger proper invalidation
131 // for them.
132 UR.CrossSCCPA.intersect(PA);
133
134 // Invalidation was handled after each pass in the above loop for the current
135 // SCC. Therefore, the remaining analysis results in the AnalysisManager are
136 // preserved. We mark this with a set so that we don't need to inspect each
137 // one individually.
138 PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
139
140 if (DebugLogging)
141 dbgs() << "Finished CGSCC pass manager run.\n";
142
143 return PA;
144 }
145
invalidate(Module & M,const PreservedAnalyses & PA,ModuleAnalysisManager::Invalidator & Inv)146 bool CGSCCAnalysisManagerModuleProxy::Result::invalidate(
147 Module &M, const PreservedAnalyses &PA,
148 ModuleAnalysisManager::Invalidator &Inv) {
149 // If literally everything is preserved, we're done.
150 if (PA.areAllPreserved())
151 return false; // This is still a valid proxy.
152
153 // If this proxy or the call graph is going to be invalidated, we also need
154 // to clear all the keys coming from that analysis.
155 //
156 // We also directly invalidate the FAM's module proxy if necessary, and if
157 // that proxy isn't preserved we can't preserve this proxy either. We rely on
158 // it to handle module -> function analysis invalidation in the face of
159 // structural changes and so if it's unavailable we conservatively clear the
160 // entire SCC layer as well rather than trying to do invalidation ourselves.
161 auto PAC = PA.getChecker<CGSCCAnalysisManagerModuleProxy>();
162 if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()) ||
163 Inv.invalidate<LazyCallGraphAnalysis>(M, PA) ||
164 Inv.invalidate<FunctionAnalysisManagerModuleProxy>(M, PA)) {
165 InnerAM->clear();
166
167 // And the proxy itself should be marked as invalid so that we can observe
168 // the new call graph. This isn't strictly necessary because we cheat
169 // above, but is still useful.
170 return true;
171 }
172
173 // Directly check if the relevant set is preserved so we can short circuit
174 // invalidating SCCs below.
175 bool AreSCCAnalysesPreserved =
176 PA.allAnalysesInSetPreserved<AllAnalysesOn<LazyCallGraph::SCC>>();
177
178 // Ok, we have a graph, so we can propagate the invalidation down into it.
179 G->buildRefSCCs();
180 for (auto &RC : G->postorder_ref_sccs())
181 for (auto &C : RC) {
182 Optional<PreservedAnalyses> InnerPA;
183
184 // Check to see whether the preserved set needs to be adjusted based on
185 // module-level analysis invalidation triggering deferred invalidation
186 // for this SCC.
187 if (auto *OuterProxy =
188 InnerAM->getCachedResult<ModuleAnalysisManagerCGSCCProxy>(C))
189 for (const auto &OuterInvalidationPair :
190 OuterProxy->getOuterInvalidations()) {
191 AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
192 const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
193 if (Inv.invalidate(OuterAnalysisID, M, PA)) {
194 if (!InnerPA)
195 InnerPA = PA;
196 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
197 InnerPA->abandon(InnerAnalysisID);
198 }
199 }
200
201 // Check if we needed a custom PA set. If so we'll need to run the inner
202 // invalidation.
203 if (InnerPA) {
204 InnerAM->invalidate(C, *InnerPA);
205 continue;
206 }
207
208 // Otherwise we only need to do invalidation if the original PA set didn't
209 // preserve all SCC analyses.
210 if (!AreSCCAnalysesPreserved)
211 InnerAM->invalidate(C, PA);
212 }
213
214 // Return false to indicate that this result is still a valid proxy.
215 return false;
216 }
217
218 template <>
219 CGSCCAnalysisManagerModuleProxy::Result
run(Module & M,ModuleAnalysisManager & AM)220 CGSCCAnalysisManagerModuleProxy::run(Module &M, ModuleAnalysisManager &AM) {
221 // Force the Function analysis manager to also be available so that it can
222 // be accessed in an SCC analysis and proxied onward to function passes.
223 // FIXME: It is pretty awkward to just drop the result here and assert that
224 // we can find it again later.
225 (void)AM.getResult<FunctionAnalysisManagerModuleProxy>(M);
226
227 return Result(*InnerAM, AM.getResult<LazyCallGraphAnalysis>(M));
228 }
229
230 AnalysisKey FunctionAnalysisManagerCGSCCProxy::Key;
231
232 FunctionAnalysisManagerCGSCCProxy::Result
run(LazyCallGraph::SCC & C,CGSCCAnalysisManager & AM,LazyCallGraph & CG)233 FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C,
234 CGSCCAnalysisManager &AM,
235 LazyCallGraph &CG) {
236 // Note: unconditionally getting checking that the proxy exists may get it at
237 // this point. There are cases when this is being run unnecessarily, but
238 // it is cheap and having the assertion in place is more valuable.
239 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerCGSCCProxy>(C, CG);
240 Module &M = *C.begin()->getFunction().getParent();
241 bool ProxyExists =
242 MAMProxy.cachedResultExists<FunctionAnalysisManagerModuleProxy>(M);
243 assert(ProxyExists &&
244 "The CGSCC pass manager requires that the FAM module proxy is run "
245 "on the module prior to entering the CGSCC walk");
246 (void)ProxyExists;
247
248 // We just return an empty result. The caller will use the updateFAM interface
249 // to correctly register the relevant FunctionAnalysisManager based on the
250 // context in which this proxy is run.
251 return Result();
252 }
253
invalidate(LazyCallGraph::SCC & C,const PreservedAnalyses & PA,CGSCCAnalysisManager::Invalidator & Inv)254 bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
255 LazyCallGraph::SCC &C, const PreservedAnalyses &PA,
256 CGSCCAnalysisManager::Invalidator &Inv) {
257 // If literally everything is preserved, we're done.
258 if (PA.areAllPreserved())
259 return false; // This is still a valid proxy.
260
261 // All updates to preserve valid results are done below, so we don't need to
262 // invalidate this proxy.
263 //
264 // Note that in order to preserve this proxy, a module pass must ensure that
265 // the FAM has been completely updated to handle the deletion of functions.
266 // Specifically, any FAM-cached results for those functions need to have been
267 // forcibly cleared. When preserved, this proxy will only invalidate results
268 // cached on functions *still in the module* at the end of the module pass.
269 auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>();
270 if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) {
271 for (LazyCallGraph::Node &N : C)
272 FAM->clear(N.getFunction(), N.getFunction().getName());
273
274 return false;
275 }
276
277 // Directly check if the relevant set is preserved.
278 bool AreFunctionAnalysesPreserved =
279 PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>();
280
281 // Now walk all the functions to see if any inner analysis invalidation is
282 // necessary.
283 for (LazyCallGraph::Node &N : C) {
284 Function &F = N.getFunction();
285 Optional<PreservedAnalyses> FunctionPA;
286
287 // Check to see whether the preserved set needs to be pruned based on
288 // SCC-level analysis invalidation that triggers deferred invalidation
289 // registered with the outer analysis manager proxy for this function.
290 if (auto *OuterProxy =
291 FAM->getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F))
292 for (const auto &OuterInvalidationPair :
293 OuterProxy->getOuterInvalidations()) {
294 AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
295 const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
296 if (Inv.invalidate(OuterAnalysisID, C, PA)) {
297 if (!FunctionPA)
298 FunctionPA = PA;
299 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
300 FunctionPA->abandon(InnerAnalysisID);
301 }
302 }
303
304 // Check if we needed a custom PA set, and if so we'll need to run the
305 // inner invalidation.
306 if (FunctionPA) {
307 FAM->invalidate(F, *FunctionPA);
308 continue;
309 }
310
311 // Otherwise we only need to do invalidation if the original PA set didn't
312 // preserve all function analyses.
313 if (!AreFunctionAnalysesPreserved)
314 FAM->invalidate(F, PA);
315 }
316
317 // Return false to indicate that this result is still a valid proxy.
318 return false;
319 }
320
321 } // end namespace llvm
322
323 /// When a new SCC is created for the graph we first update the
324 /// FunctionAnalysisManager in the Proxy's result.
325 /// As there might be function analysis results cached for the functions now in
326 /// that SCC, two forms of updates are required.
327 ///
328 /// First, a proxy from the SCC to the FunctionAnalysisManager needs to be
329 /// created so that any subsequent invalidation events to the SCC are
330 /// propagated to the function analysis results cached for functions within it.
331 ///
332 /// Second, if any of the functions within the SCC have analysis results with
333 /// outer analysis dependencies, then those dependencies would point to the
334 /// *wrong* SCC's analysis result. We forcibly invalidate the necessary
335 /// function analyses so that they don't retain stale handles.
updateNewSCCFunctionAnalyses(LazyCallGraph::SCC & C,LazyCallGraph & G,CGSCCAnalysisManager & AM,FunctionAnalysisManager & FAM)336 static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C,
337 LazyCallGraph &G,
338 CGSCCAnalysisManager &AM,
339 FunctionAnalysisManager &FAM) {
340 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, G).updateFAM(FAM);
341
342 // Now walk the functions in this SCC and invalidate any function analysis
343 // results that might have outer dependencies on an SCC analysis.
344 for (LazyCallGraph::Node &N : C) {
345 Function &F = N.getFunction();
346
347 auto *OuterProxy =
348 FAM.getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F);
349 if (!OuterProxy)
350 // No outer analyses were queried, nothing to do.
351 continue;
352
353 // Forcibly abandon all the inner analyses with dependencies, but
354 // invalidate nothing else.
355 auto PA = PreservedAnalyses::all();
356 for (const auto &OuterInvalidationPair :
357 OuterProxy->getOuterInvalidations()) {
358 const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
359 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
360 PA.abandon(InnerAnalysisID);
361 }
362
363 // Now invalidate anything we found.
364 FAM.invalidate(F, PA);
365 }
366 }
367
368 /// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c
369 /// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly
370 /// added SCCs.
371 ///
372 /// The range of new SCCs must be in postorder already. The SCC they were split
373 /// out of must be provided as \p C. The current node being mutated and
374 /// triggering updates must be passed as \p N.
375 ///
376 /// This function returns the SCC containing \p N. This will be either \p C if
377 /// no new SCCs have been split out, or it will be the new SCC containing \p N.
378 template <typename SCCRangeT>
379 static LazyCallGraph::SCC *
incorporateNewSCCRange(const SCCRangeT & NewSCCRange,LazyCallGraph & G,LazyCallGraph::Node & N,LazyCallGraph::SCC * C,CGSCCAnalysisManager & AM,CGSCCUpdateResult & UR)380 incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G,
381 LazyCallGraph::Node &N, LazyCallGraph::SCC *C,
382 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) {
383 using SCC = LazyCallGraph::SCC;
384
385 if (NewSCCRange.begin() == NewSCCRange.end())
386 return C;
387
388 // Add the current SCC to the worklist as its shape has changed.
389 UR.CWorklist.insert(C);
390 LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist:" << *C
391 << "\n");
392
393 SCC *OldC = C;
394
395 // Update the current SCC. Note that if we have new SCCs, this must actually
396 // change the SCC.
397 assert(C != &*NewSCCRange.begin() &&
398 "Cannot insert new SCCs without changing current SCC!");
399 C = &*NewSCCRange.begin();
400 assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
401
402 // If we had a cached FAM proxy originally, we will want to create more of
403 // them for each SCC that was split off.
404 FunctionAnalysisManager *FAM = nullptr;
405 if (auto *FAMProxy =
406 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*OldC))
407 FAM = &FAMProxy->getManager();
408
409 // We need to propagate an invalidation call to all but the newly current SCC
410 // because the outer pass manager won't do that for us after splitting them.
411 // FIXME: We should accept a PreservedAnalysis from the CG updater so that if
412 // there are preserved analysis we can avoid invalidating them here for
413 // split-off SCCs.
414 // We know however that this will preserve any FAM proxy so go ahead and mark
415 // that.
416 PreservedAnalyses PA;
417 PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
418 AM.invalidate(*OldC, PA);
419
420 // Ensure the now-current SCC's function analyses are updated.
421 if (FAM)
422 updateNewSCCFunctionAnalyses(*C, G, AM, *FAM);
423
424 for (SCC &NewC : llvm::reverse(make_range(std::next(NewSCCRange.begin()),
425 NewSCCRange.end()))) {
426 assert(C != &NewC && "No need to re-visit the current SCC!");
427 assert(OldC != &NewC && "Already handled the original SCC!");
428 UR.CWorklist.insert(&NewC);
429 LLVM_DEBUG(dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n");
430
431 // Ensure new SCCs' function analyses are updated.
432 if (FAM)
433 updateNewSCCFunctionAnalyses(NewC, G, AM, *FAM);
434
435 // Also propagate a normal invalidation to the new SCC as only the current
436 // will get one from the pass manager infrastructure.
437 AM.invalidate(NewC, PA);
438 }
439 return C;
440 }
441
updateCGAndAnalysisManagerForPass(LazyCallGraph & G,LazyCallGraph::SCC & InitialC,LazyCallGraph::Node & N,CGSCCAnalysisManager & AM,CGSCCUpdateResult & UR,FunctionAnalysisManager & FAM,bool FunctionPass)442 static LazyCallGraph::SCC &updateCGAndAnalysisManagerForPass(
443 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
444 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
445 FunctionAnalysisManager &FAM, bool FunctionPass) {
446 using Node = LazyCallGraph::Node;
447 using Edge = LazyCallGraph::Edge;
448 using SCC = LazyCallGraph::SCC;
449 using RefSCC = LazyCallGraph::RefSCC;
450
451 RefSCC &InitialRC = InitialC.getOuterRefSCC();
452 SCC *C = &InitialC;
453 RefSCC *RC = &InitialRC;
454 Function &F = N.getFunction();
455
456 // Walk the function body and build up the set of retained, promoted, and
457 // demoted edges.
458 SmallVector<Constant *, 16> Worklist;
459 SmallPtrSet<Constant *, 16> Visited;
460 SmallPtrSet<Node *, 16> RetainedEdges;
461 SmallSetVector<Node *, 4> PromotedRefTargets;
462 SmallSetVector<Node *, 4> DemotedCallTargets;
463 SmallSetVector<Node *, 4> NewCallEdges;
464 SmallSetVector<Node *, 4> NewRefEdges;
465
466 // First walk the function and handle all called functions. We do this first
467 // because if there is a single call edge, whether there are ref edges is
468 // irrelevant.
469 for (Instruction &I : instructions(F))
470 if (auto *CB = dyn_cast<CallBase>(&I))
471 if (Function *Callee = CB->getCalledFunction())
472 if (Visited.insert(Callee).second && !Callee->isDeclaration()) {
473 Node &CalleeN = *G.lookup(*Callee);
474 Edge *E = N->lookup(CalleeN);
475 assert((E || !FunctionPass) &&
476 "No function transformations should introduce *new* "
477 "call edges! Any new calls should be modeled as "
478 "promoted existing ref edges!");
479 bool Inserted = RetainedEdges.insert(&CalleeN).second;
480 (void)Inserted;
481 assert(Inserted && "We should never visit a function twice.");
482 if (!E)
483 NewCallEdges.insert(&CalleeN);
484 else if (!E->isCall())
485 PromotedRefTargets.insert(&CalleeN);
486 }
487
488 // Now walk all references.
489 for (Instruction &I : instructions(F))
490 for (Value *Op : I.operand_values())
491 if (auto *C = dyn_cast<Constant>(Op))
492 if (Visited.insert(C).second)
493 Worklist.push_back(C);
494
495 auto VisitRef = [&](Function &Referee) {
496 Node &RefereeN = *G.lookup(Referee);
497 Edge *E = N->lookup(RefereeN);
498 assert((E || !FunctionPass) &&
499 "No function transformations should introduce *new* ref "
500 "edges! Any new ref edges would require IPO which "
501 "function passes aren't allowed to do!");
502 bool Inserted = RetainedEdges.insert(&RefereeN).second;
503 (void)Inserted;
504 assert(Inserted && "We should never visit a function twice.");
505 if (!E)
506 NewRefEdges.insert(&RefereeN);
507 else if (E->isCall())
508 DemotedCallTargets.insert(&RefereeN);
509 };
510 LazyCallGraph::visitReferences(Worklist, Visited, VisitRef);
511
512 // Handle new ref edges.
513 for (Node *RefTarget : NewRefEdges) {
514 SCC &TargetC = *G.lookupSCC(*RefTarget);
515 RefSCC &TargetRC = TargetC.getOuterRefSCC();
516 (void)TargetRC;
517 // TODO: This only allows trivial edges to be added for now.
518 assert((RC == &TargetRC ||
519 RC->isAncestorOf(TargetRC)) && "New ref edge is not trivial!");
520 RC->insertTrivialRefEdge(N, *RefTarget);
521 }
522
523 // Handle new call edges.
524 for (Node *CallTarget : NewCallEdges) {
525 SCC &TargetC = *G.lookupSCC(*CallTarget);
526 RefSCC &TargetRC = TargetC.getOuterRefSCC();
527 (void)TargetRC;
528 // TODO: This only allows trivial edges to be added for now.
529 assert((RC == &TargetRC ||
530 RC->isAncestorOf(TargetRC)) && "New call edge is not trivial!");
531 RC->insertTrivialCallEdge(N, *CallTarget);
532 }
533
534 // Include synthetic reference edges to known, defined lib functions.
535 for (auto *F : G.getLibFunctions())
536 // While the list of lib functions doesn't have repeats, don't re-visit
537 // anything handled above.
538 if (!Visited.count(F))
539 VisitRef(*F);
540
541 // First remove all of the edges that are no longer present in this function.
542 // The first step makes these edges uniformly ref edges and accumulates them
543 // into a separate data structure so removal doesn't invalidate anything.
544 SmallVector<Node *, 4> DeadTargets;
545 for (Edge &E : *N) {
546 if (RetainedEdges.count(&E.getNode()))
547 continue;
548
549 SCC &TargetC = *G.lookupSCC(E.getNode());
550 RefSCC &TargetRC = TargetC.getOuterRefSCC();
551 if (&TargetRC == RC && E.isCall()) {
552 if (C != &TargetC) {
553 // For separate SCCs this is trivial.
554 RC->switchTrivialInternalEdgeToRef(N, E.getNode());
555 } else {
556 // Now update the call graph.
557 C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, E.getNode()),
558 G, N, C, AM, UR);
559 }
560 }
561
562 // Now that this is ready for actual removal, put it into our list.
563 DeadTargets.push_back(&E.getNode());
564 }
565 // Remove the easy cases quickly and actually pull them out of our list.
566 DeadTargets.erase(
567 llvm::remove_if(DeadTargets,
568 [&](Node *TargetN) {
569 SCC &TargetC = *G.lookupSCC(*TargetN);
570 RefSCC &TargetRC = TargetC.getOuterRefSCC();
571
572 // We can't trivially remove internal targets, so skip
573 // those.
574 if (&TargetRC == RC)
575 return false;
576
577 RC->removeOutgoingEdge(N, *TargetN);
578 LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '"
579 << N << "' to '" << TargetN << "'\n");
580 return true;
581 }),
582 DeadTargets.end());
583
584 // Now do a batch removal of the internal ref edges left.
585 auto NewRefSCCs = RC->removeInternalRefEdge(N, DeadTargets);
586 if (!NewRefSCCs.empty()) {
587 // The old RefSCC is dead, mark it as such.
588 UR.InvalidatedRefSCCs.insert(RC);
589
590 // Note that we don't bother to invalidate analyses as ref-edge
591 // connectivity is not really observable in any way and is intended
592 // exclusively to be used for ordering of transforms rather than for
593 // analysis conclusions.
594
595 // Update RC to the "bottom".
596 assert(G.lookupSCC(N) == C && "Changed the SCC when splitting RefSCCs!");
597 RC = &C->getOuterRefSCC();
598 assert(G.lookupRefSCC(N) == RC && "Failed to update current RefSCC!");
599
600 // The RC worklist is in reverse postorder, so we enqueue the new ones in
601 // RPO except for the one which contains the source node as that is the
602 // "bottom" we will continue processing in the bottom-up walk.
603 assert(NewRefSCCs.front() == RC &&
604 "New current RefSCC not first in the returned list!");
605 for (RefSCC *NewRC : llvm::reverse(make_range(std::next(NewRefSCCs.begin()),
606 NewRefSCCs.end()))) {
607 assert(NewRC != RC && "Should not encounter the current RefSCC further "
608 "in the postorder list of new RefSCCs.");
609 UR.RCWorklist.insert(NewRC);
610 LLVM_DEBUG(dbgs() << "Enqueuing a new RefSCC in the update worklist: "
611 << *NewRC << "\n");
612 }
613 }
614
615 // Next demote all the call edges that are now ref edges. This helps make
616 // the SCCs small which should minimize the work below as we don't want to
617 // form cycles that this would break.
618 for (Node *RefTarget : DemotedCallTargets) {
619 SCC &TargetC = *G.lookupSCC(*RefTarget);
620 RefSCC &TargetRC = TargetC.getOuterRefSCC();
621
622 // The easy case is when the target RefSCC is not this RefSCC. This is
623 // only supported when the target RefSCC is a child of this RefSCC.
624 if (&TargetRC != RC) {
625 assert(RC->isAncestorOf(TargetRC) &&
626 "Cannot potentially form RefSCC cycles here!");
627 RC->switchOutgoingEdgeToRef(N, *RefTarget);
628 LLVM_DEBUG(dbgs() << "Switch outgoing call edge to a ref edge from '" << N
629 << "' to '" << *RefTarget << "'\n");
630 continue;
631 }
632
633 // We are switching an internal call edge to a ref edge. This may split up
634 // some SCCs.
635 if (C != &TargetC) {
636 // For separate SCCs this is trivial.
637 RC->switchTrivialInternalEdgeToRef(N, *RefTarget);
638 continue;
639 }
640
641 // Now update the call graph.
642 C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N,
643 C, AM, UR);
644 }
645
646 // Now promote ref edges into call edges.
647 for (Node *CallTarget : PromotedRefTargets) {
648 SCC &TargetC = *G.lookupSCC(*CallTarget);
649 RefSCC &TargetRC = TargetC.getOuterRefSCC();
650
651 // The easy case is when the target RefSCC is not this RefSCC. This is
652 // only supported when the target RefSCC is a child of this RefSCC.
653 if (&TargetRC != RC) {
654 assert(RC->isAncestorOf(TargetRC) &&
655 "Cannot potentially form RefSCC cycles here!");
656 RC->switchOutgoingEdgeToCall(N, *CallTarget);
657 LLVM_DEBUG(dbgs() << "Switch outgoing ref edge to a call edge from '" << N
658 << "' to '" << *CallTarget << "'\n");
659 continue;
660 }
661 LLVM_DEBUG(dbgs() << "Switch an internal ref edge to a call edge from '"
662 << N << "' to '" << *CallTarget << "'\n");
663
664 // Otherwise we are switching an internal ref edge to a call edge. This
665 // may merge away some SCCs, and we add those to the UpdateResult. We also
666 // need to make sure to update the worklist in the event SCCs have moved
667 // before the current one in the post-order sequence
668 bool HasFunctionAnalysisProxy = false;
669 auto InitialSCCIndex = RC->find(*C) - RC->begin();
670 bool FormedCycle = RC->switchInternalEdgeToCall(
671 N, *CallTarget, [&](ArrayRef<SCC *> MergedSCCs) {
672 for (SCC *MergedC : MergedSCCs) {
673 assert(MergedC != &TargetC && "Cannot merge away the target SCC!");
674
675 HasFunctionAnalysisProxy |=
676 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(
677 *MergedC) != nullptr;
678
679 // Mark that this SCC will no longer be valid.
680 UR.InvalidatedSCCs.insert(MergedC);
681
682 // FIXME: We should really do a 'clear' here to forcibly release
683 // memory, but we don't have a good way of doing that and
684 // preserving the function analyses.
685 auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
686 PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
687 AM.invalidate(*MergedC, PA);
688 }
689 });
690
691 // If we formed a cycle by creating this call, we need to update more data
692 // structures.
693 if (FormedCycle) {
694 C = &TargetC;
695 assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
696
697 // If one of the invalidated SCCs had a cached proxy to a function
698 // analysis manager, we need to create a proxy in the new current SCC as
699 // the invalidated SCCs had their functions moved.
700 if (HasFunctionAnalysisProxy)
701 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G).updateFAM(FAM);
702
703 // Any analyses cached for this SCC are no longer precise as the shape
704 // has changed by introducing this cycle. However, we have taken care to
705 // update the proxies so it remains valide.
706 auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
707 PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
708 AM.invalidate(*C, PA);
709 }
710 auto NewSCCIndex = RC->find(*C) - RC->begin();
711 // If we have actually moved an SCC to be topologically "below" the current
712 // one due to merging, we will need to revisit the current SCC after
713 // visiting those moved SCCs.
714 //
715 // It is critical that we *do not* revisit the current SCC unless we
716 // actually move SCCs in the process of merging because otherwise we may
717 // form a cycle where an SCC is split apart, merged, split, merged and so
718 // on infinitely.
719 if (InitialSCCIndex < NewSCCIndex) {
720 // Put our current SCC back onto the worklist as we'll visit other SCCs
721 // that are now definitively ordered prior to the current one in the
722 // post-order sequence, and may end up observing more precise context to
723 // optimize the current SCC.
724 UR.CWorklist.insert(C);
725 LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist: " << *C
726 << "\n");
727 // Enqueue in reverse order as we pop off the back of the worklist.
728 for (SCC &MovedC : llvm::reverse(make_range(RC->begin() + InitialSCCIndex,
729 RC->begin() + NewSCCIndex))) {
730 UR.CWorklist.insert(&MovedC);
731 LLVM_DEBUG(dbgs() << "Enqueuing a newly earlier in post-order SCC: "
732 << MovedC << "\n");
733 }
734 }
735 }
736
737 assert(!UR.InvalidatedSCCs.count(C) && "Invalidated the current SCC!");
738 assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!");
739 assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!");
740
741 // Record the current RefSCC and SCC for higher layers of the CGSCC pass
742 // manager now that all the updates have been applied.
743 if (RC != &InitialRC)
744 UR.UpdatedRC = RC;
745 if (C != &InitialC)
746 UR.UpdatedC = C;
747
748 return *C;
749 }
750
updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph & G,LazyCallGraph::SCC & InitialC,LazyCallGraph::Node & N,CGSCCAnalysisManager & AM,CGSCCUpdateResult & UR,FunctionAnalysisManager & FAM)751 LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass(
752 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
753 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
754 FunctionAnalysisManager &FAM) {
755 return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM,
756 /* FunctionPass */ true);
757 }
updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph & G,LazyCallGraph::SCC & InitialC,LazyCallGraph::Node & N,CGSCCAnalysisManager & AM,CGSCCUpdateResult & UR,FunctionAnalysisManager & FAM)758 LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForCGSCCPass(
759 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
760 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
761 FunctionAnalysisManager &FAM) {
762 return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM,
763 /* FunctionPass */ false);
764 }
765