1 /* 2 * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.phases.common.inlining.walker; 26 27 import java.util.ArrayList; 28 import java.util.function.ToDoubleFunction; 29 30 import jdk.internal.vm.compiler.collections.EconomicMap; 31 import jdk.internal.vm.compiler.collections.Equivalence; 32 import org.graalvm.compiler.core.common.SuppressFBWarnings; 33 import org.graalvm.compiler.graph.Node; 34 import org.graalvm.compiler.graph.NodeWorkList; 35 import org.graalvm.compiler.nodes.AbstractBeginNode; 36 import org.graalvm.compiler.nodes.AbstractMergeNode; 37 import org.graalvm.compiler.nodes.ControlSinkNode; 38 import org.graalvm.compiler.nodes.ControlSplitNode; 39 import org.graalvm.compiler.nodes.EndNode; 40 import org.graalvm.compiler.nodes.FixedNode; 41 import org.graalvm.compiler.nodes.FixedWithNextNode; 42 import org.graalvm.compiler.nodes.Invoke; 43 import org.graalvm.compiler.nodes.LoopBeginNode; 44 import org.graalvm.compiler.nodes.LoopEndNode; 45 import org.graalvm.compiler.nodes.LoopExitNode; 46 import org.graalvm.compiler.nodes.MergeNode; 47 import org.graalvm.compiler.nodes.StartNode; 48 import org.graalvm.compiler.nodes.StructuredGraph; 49 import org.graalvm.compiler.phases.common.inlining.InliningUtil; 50 51 public class ComputeInliningRelevance { 52 53 private static final double EPSILON = 1d / Integer.MAX_VALUE; 54 private static final double UNINITIALIZED = -1D; 55 56 private static final int EXPECTED_MIN_INVOKE_COUNT = 3; 57 private static final int EXPECTED_INVOKE_RATIO = 20; 58 private static final int EXPECTED_LOOP_COUNT = 3; 59 60 private final StructuredGraph graph; 61 private final ToDoubleFunction<FixedNode> nodeProbabilities; 62 63 /** 64 * Node relevances are pre-computed for all invokes if the graph contains loops. If there are no 65 * loops, the computation happens lazily based on {@link #rootScope}. 66 */ 67 private EconomicMap<FixedNode, Double> nodeRelevances; 68 /** 69 * This scope is non-null if (and only if) there are no loops in the graph. In this case, the 70 * root scope is used to compute invoke relevances on the fly. 71 */ 72 private Scope rootScope; 73 ComputeInliningRelevance(StructuredGraph graph, ToDoubleFunction<FixedNode> nodeProbabilities)74 public ComputeInliningRelevance(StructuredGraph graph, ToDoubleFunction<FixedNode> nodeProbabilities) { 75 this.graph = graph; 76 this.nodeProbabilities = nodeProbabilities; 77 } 78 79 /** 80 * Initializes or updates the relevance computation. If there are no loops within the graph, 81 * most computation happens lazily. 82 */ compute()83 public void compute() { 84 rootScope = null; 85 if (!graph.hasLoops()) { 86 // fast path for the frequent case of no loops 87 rootScope = new Scope(graph.start(), null); 88 } else { 89 if (nodeRelevances == null) { 90 nodeRelevances = EconomicMap.create(Equivalence.IDENTITY, EXPECTED_MIN_INVOKE_COUNT + InliningUtil.getNodeCount(graph) / EXPECTED_INVOKE_RATIO); 91 } 92 NodeWorkList workList = graph.createNodeWorkList(); 93 EconomicMap<LoopBeginNode, Scope> loops = EconomicMap.create(Equivalence.IDENTITY, EXPECTED_LOOP_COUNT); 94 95 Scope topScope = new Scope(graph.start(), null); 96 for (LoopBeginNode loopBegin : graph.getNodes(LoopBeginNode.TYPE)) { 97 createLoopScope(loopBegin, loops, topScope); 98 } 99 100 topScope.process(workList); 101 for (Scope scope : loops.getValues()) { 102 scope.process(workList); 103 } 104 } 105 } 106 getRelevance(Invoke invoke)107 public double getRelevance(Invoke invoke) { 108 if (rootScope != null) { 109 return rootScope.computeInvokeRelevance(invoke); 110 } 111 assert nodeRelevances != null : "uninitialized relevance"; 112 return nodeRelevances.get(invoke.asNode()); 113 } 114 115 /** 116 * Determines the parent of the given loop and creates a {@link Scope} object for each one. This 117 * method will call itself recursively if no {@link Scope} for the parent loop exists. 118 */ createLoopScope(LoopBeginNode loopBegin, EconomicMap<LoopBeginNode, Scope> loops, Scope topScope)119 private Scope createLoopScope(LoopBeginNode loopBegin, EconomicMap<LoopBeginNode, Scope> loops, Scope topScope) { 120 Scope scope = loops.get(loopBegin); 121 if (scope == null) { 122 final Scope parent; 123 // look for the parent scope 124 FixedNode current = loopBegin.forwardEnd(); 125 while (true) { 126 if (current.predecessor() == null) { 127 if (current instanceof LoopBeginNode) { 128 // if we reach a LoopBeginNode then we're within this loop 129 parent = createLoopScope((LoopBeginNode) current, loops, topScope); 130 break; 131 } else if (current instanceof StartNode) { 132 // we're within the outermost scope 133 parent = topScope; 134 break; 135 } else { 136 assert current instanceof MergeNode : current; 137 // follow any path upwards - it doesn't matter which one 138 current = ((AbstractMergeNode) current).forwardEndAt(0); 139 } 140 } else if (current instanceof LoopExitNode) { 141 // if we reach a loop exit then we follow this loop and have the same parent 142 parent = createLoopScope(((LoopExitNode) current).loopBegin(), loops, topScope).parent; 143 break; 144 } else { 145 current = (FixedNode) current.predecessor(); 146 } 147 } 148 scope = new Scope(loopBegin, parent); 149 loops.put(loopBegin, scope); 150 } 151 return scope; 152 } 153 154 /** 155 * A scope holds information for the contents of one loop or of the root of the method. It does 156 * not include child loops, i.e., the iteration in {@link #process(NodeWorkList)} explicitly 157 * excludes the nodes of child loops. 158 */ 159 private class Scope { 160 public final FixedNode start; 161 public final Scope parent; // can be null for the outermost scope 162 163 /** 164 * The minimum probability along the most probable path in this scope. Computed lazily. 165 */ 166 private double fastPathMinProbability = UNINITIALIZED; 167 /** 168 * A measure of how important this scope is within its parent scope. Computed lazily. 169 */ 170 private double scopeRelevanceWithinParent = UNINITIALIZED; 171 Scope(FixedNode start, Scope parent)172 Scope(FixedNode start, Scope parent) { 173 this.start = start; 174 this.parent = parent; 175 } 176 177 @SuppressFBWarnings(value = "FE_FLOATING_POINT_EQUALITY", justification = "comparing against -1D is accurate") getFastPathMinProbability()178 public double getFastPathMinProbability() { 179 if (fastPathMinProbability == UNINITIALIZED) { 180 fastPathMinProbability = Math.max(EPSILON, computeFastPathMinProbability(start)); 181 } 182 return fastPathMinProbability; 183 } 184 185 /** 186 * Computes the ratio between the probabilities of the current scope's entry point and the 187 * parent scope's fastPathMinProbability. 188 */ 189 @SuppressFBWarnings(value = "FE_FLOATING_POINT_EQUALITY", justification = "comparing against -1D is accurate") getScopeRelevanceWithinParent()190 public double getScopeRelevanceWithinParent() { 191 if (scopeRelevanceWithinParent == UNINITIALIZED) { 192 if (start instanceof LoopBeginNode) { 193 assert parent != null; 194 double scopeEntryProbability = nodeProbabilities.applyAsDouble(((LoopBeginNode) start).forwardEnd()); 195 196 scopeRelevanceWithinParent = scopeEntryProbability / parent.getFastPathMinProbability(); 197 } else { 198 scopeRelevanceWithinParent = 1D; 199 } 200 } 201 return scopeRelevanceWithinParent; 202 } 203 204 /** 205 * Processes all invokes in this scope by starting at the scope's start node and iterating 206 * all fixed nodes. Child loops are skipped by going from loop entries directly to the loop 207 * exits. Processing stops at loop exits of the current loop. 208 */ process(NodeWorkList workList)209 public void process(NodeWorkList workList) { 210 assert !(start instanceof Invoke); 211 workList.addAll(start.successors()); 212 213 for (Node current : workList) { 214 assert current.isAlive(); 215 216 if (current instanceof Invoke) { 217 // process the invoke and queue its successors 218 nodeRelevances.put((FixedNode) current, computeInvokeRelevance((Invoke) current)); 219 workList.addAll(current.successors()); 220 } else if (current instanceof LoopBeginNode) { 221 // skip child loops by advancing over the loop exits 222 ((LoopBeginNode) current).loopExits().forEach(exit -> workList.add(exit.next())); 223 } else if (current instanceof LoopEndNode) { 224 // nothing to do 225 } else if (current instanceof LoopExitNode) { 226 // nothing to do 227 } else if (current instanceof FixedWithNextNode) { 228 workList.add(((FixedWithNextNode) current).next()); 229 } else if (current instanceof EndNode) { 230 workList.add(((EndNode) current).merge()); 231 } else if (current instanceof ControlSinkNode) { 232 // nothing to do 233 } else if (current instanceof ControlSplitNode) { 234 workList.addAll(current.successors()); 235 } else { 236 assert false : current; 237 } 238 } 239 } 240 241 /** 242 * The relevance of an invoke is the ratio between the invoke's probability and the current 243 * scope's fastPathMinProbability, adjusted by scopeRelevanceWithinParent. 244 */ computeInvokeRelevance(Invoke invoke)245 public double computeInvokeRelevance(Invoke invoke) { 246 double invokeProbability = nodeProbabilities.applyAsDouble(invoke.asNode()); 247 assert !Double.isNaN(invokeProbability); 248 249 double relevance = (invokeProbability / getFastPathMinProbability()) * Math.min(1.0, getScopeRelevanceWithinParent()); 250 assert !Double.isNaN(relevance) : invoke + ": " + relevance + " / " + invokeProbability + " / " + getFastPathMinProbability() + " / " + getScopeRelevanceWithinParent(); 251 return relevance; 252 } 253 } 254 255 /** 256 * Computes the minimum probability along the most probable path within the scope. During 257 * iteration, the method returns immediately once a loop exit is discovered. 258 */ computeFastPathMinProbability(FixedNode scopeStart)259 private double computeFastPathMinProbability(FixedNode scopeStart) { 260 ArrayList<FixedNode> pathBeginNodes = new ArrayList<>(); 261 pathBeginNodes.add(scopeStart); 262 double minPathProbability = nodeProbabilities.applyAsDouble(scopeStart); 263 boolean isLoopScope = scopeStart instanceof LoopBeginNode; 264 265 do { 266 Node current = pathBeginNodes.remove(pathBeginNodes.size() - 1); 267 do { 268 if (isLoopScope && current instanceof LoopExitNode && ((LoopBeginNode) scopeStart).loopExits().contains((LoopExitNode) current)) { 269 return minPathProbability; 270 } else if (current instanceof LoopBeginNode && current != scopeStart) { 271 current = getMaxProbabilityLoopExit((LoopBeginNode) current, pathBeginNodes); 272 minPathProbability = getMinPathProbability((FixedNode) current, minPathProbability); 273 } else if (current instanceof ControlSplitNode) { 274 current = getMaxProbabilitySux((ControlSplitNode) current, pathBeginNodes); 275 minPathProbability = getMinPathProbability((FixedNode) current, minPathProbability); 276 } else { 277 assert current.successors().count() <= 1; 278 current = current.successors().first(); 279 } 280 } while (current != null); 281 } while (!pathBeginNodes.isEmpty()); 282 283 return minPathProbability; 284 } 285 getMinPathProbability(FixedNode current, double minPathProbability)286 private double getMinPathProbability(FixedNode current, double minPathProbability) { 287 return current == null ? minPathProbability : Math.min(minPathProbability, nodeProbabilities.applyAsDouble(current)); 288 } 289 290 /** 291 * Returns the most probable successor. If multiple successors share the maximum probability, 292 * one is returned and the others are enqueued in pathBeginNodes. 293 */ getMaxProbabilitySux(ControlSplitNode controlSplit, ArrayList<FixedNode> pathBeginNodes)294 private static Node getMaxProbabilitySux(ControlSplitNode controlSplit, ArrayList<FixedNode> pathBeginNodes) { 295 Node maxSux = null; 296 double maxProbability = 0.0; 297 int pathBeginCount = pathBeginNodes.size(); 298 299 for (Node sux : controlSplit.successors()) { 300 double probability = controlSplit.probability((AbstractBeginNode) sux); 301 if (probability > maxProbability) { 302 maxProbability = probability; 303 maxSux = sux; 304 truncate(pathBeginNodes, pathBeginCount); 305 } else if (probability == maxProbability) { 306 pathBeginNodes.add((FixedNode) sux); 307 } 308 } 309 310 return maxSux; 311 } 312 313 /** 314 * Returns the most probable loop exit. If multiple successors share the maximum probability, 315 * one is returned and the others are enqueued in pathBeginNodes. 316 */ getMaxProbabilityLoopExit(LoopBeginNode loopBegin, ArrayList<FixedNode> pathBeginNodes)317 private Node getMaxProbabilityLoopExit(LoopBeginNode loopBegin, ArrayList<FixedNode> pathBeginNodes) { 318 Node maxSux = null; 319 double maxProbability = 0.0; 320 int pathBeginCount = pathBeginNodes.size(); 321 322 for (LoopExitNode sux : loopBegin.loopExits()) { 323 double probability = nodeProbabilities.applyAsDouble(sux); 324 if (probability > maxProbability) { 325 maxProbability = probability; 326 maxSux = sux; 327 truncate(pathBeginNodes, pathBeginCount); 328 } else if (probability == maxProbability) { 329 pathBeginNodes.add(sux); 330 } 331 } 332 333 return maxSux; 334 } 335 truncate(ArrayList<FixedNode> pathBeginNodes, int pathBeginCount)336 private static void truncate(ArrayList<FixedNode> pathBeginNodes, int pathBeginCount) { 337 for (int i = pathBeginNodes.size() - pathBeginCount; i > 0; i--) { 338 pathBeginNodes.remove(pathBeginNodes.size() - 1); 339 } 340 } 341 } 342