1 /* 2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements; 26 27 import static org.graalvm.compiler.debug.GraalError.unimplemented; 28 import static org.graalvm.compiler.nodeinfo.InputType.Anchor; 29 import static org.graalvm.compiler.nodeinfo.InputType.Guard; 30 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_0; 31 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; 32 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_0; 33 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED; 34 35 import java.net.URI; 36 import java.util.ArrayList; 37 import java.util.Arrays; 38 import java.util.Formatter; 39 import java.util.HashMap; 40 import java.util.List; 41 import java.util.Map; 42 43 import jdk.internal.vm.compiler.collections.EconomicMap; 44 import jdk.internal.vm.compiler.collections.Equivalence; 45 import org.graalvm.compiler.api.replacements.Fold; 46 import org.graalvm.compiler.bytecode.Bytecode; 47 import org.graalvm.compiler.bytecode.BytecodeProvider; 48 import org.graalvm.compiler.core.common.PermanentBailoutException; 49 import org.graalvm.compiler.core.common.cfg.CFGVerifier; 50 import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; 51 import org.graalvm.compiler.core.common.type.Stamp; 52 import org.graalvm.compiler.core.common.type.StampFactory; 53 import org.graalvm.compiler.core.common.type.StampPair; 54 import org.graalvm.compiler.debug.DebugCloseable; 55 import org.graalvm.compiler.debug.DebugContext; 56 import org.graalvm.compiler.debug.GraalError; 57 import org.graalvm.compiler.graph.IterableNodeType; 58 import org.graalvm.compiler.graph.Node; 59 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 60 import org.graalvm.compiler.graph.NodeClass; 61 import org.graalvm.compiler.graph.NodeSourcePosition; 62 import org.graalvm.compiler.graph.SourceLanguagePosition; 63 import org.graalvm.compiler.graph.SourceLanguagePositionProvider; 64 import org.graalvm.compiler.graph.spi.Canonicalizable; 65 import org.graalvm.compiler.java.GraphBuilderPhase; 66 import org.graalvm.compiler.nodeinfo.InputType; 67 import org.graalvm.compiler.nodeinfo.NodeInfo; 68 import org.graalvm.compiler.nodes.AbstractBeginNode; 69 import org.graalvm.compiler.nodes.AbstractMergeNode; 70 import org.graalvm.compiler.nodes.CallTargetNode; 71 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; 72 import org.graalvm.compiler.nodes.ControlSinkNode; 73 import org.graalvm.compiler.nodes.DeoptBciSupplier; 74 import org.graalvm.compiler.nodes.DeoptimizeNode; 75 import org.graalvm.compiler.nodes.EncodedGraph; 76 import org.graalvm.compiler.nodes.FixedNode; 77 import org.graalvm.compiler.nodes.FixedWithNextNode; 78 import org.graalvm.compiler.nodes.FrameState; 79 import org.graalvm.compiler.nodes.IfNode; 80 import org.graalvm.compiler.nodes.Invoke; 81 import org.graalvm.compiler.nodes.InvokeWithExceptionNode; 82 import org.graalvm.compiler.nodes.MergeNode; 83 import org.graalvm.compiler.nodes.NodeView; 84 import org.graalvm.compiler.nodes.ParameterNode; 85 import org.graalvm.compiler.nodes.PluginReplacementNode; 86 import org.graalvm.compiler.nodes.ReturnNode; 87 import org.graalvm.compiler.nodes.SimplifyingGraphDecoder; 88 import org.graalvm.compiler.nodes.StateSplit; 89 import org.graalvm.compiler.nodes.StructuredGraph; 90 import org.graalvm.compiler.nodes.UnwindNode; 91 import org.graalvm.compiler.nodes.ValueNode; 92 import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; 93 import org.graalvm.compiler.nodes.extended.AnchoringNode; 94 import org.graalvm.compiler.nodes.extended.GuardingNode; 95 import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; 96 import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin; 97 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; 98 import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; 99 import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; 100 import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; 101 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; 102 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; 103 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.InvocationPluginReceiver; 104 import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin; 105 import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind; 106 import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; 107 import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; 108 import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; 109 import org.graalvm.compiler.nodes.java.LoadFieldNode; 110 import org.graalvm.compiler.nodes.java.LoadIndexedNode; 111 import org.graalvm.compiler.nodes.java.MethodCallTargetNode; 112 import org.graalvm.compiler.nodes.java.MonitorIdNode; 113 import org.graalvm.compiler.nodes.java.NewArrayNode; 114 import org.graalvm.compiler.nodes.java.NewInstanceNode; 115 import org.graalvm.compiler.nodes.java.NewMultiArrayNode; 116 import org.graalvm.compiler.nodes.java.StoreFieldNode; 117 import org.graalvm.compiler.nodes.java.StoreIndexedNode; 118 import org.graalvm.compiler.nodes.spi.CoreProviders; 119 import org.graalvm.compiler.nodes.spi.Replacements; 120 import org.graalvm.compiler.nodes.spi.StampProvider; 121 import org.graalvm.compiler.nodes.type.StampTool; 122 import org.graalvm.compiler.nodes.util.GraphUtil; 123 import org.graalvm.compiler.options.Option; 124 import org.graalvm.compiler.options.OptionKey; 125 import org.graalvm.compiler.options.OptionType; 126 import org.graalvm.compiler.options.OptionValues; 127 import org.graalvm.compiler.phases.common.inlining.InliningUtil; 128 129 import jdk.vm.ci.code.Architecture; 130 import jdk.vm.ci.code.BailoutException; 131 import jdk.vm.ci.code.BytecodeFrame; 132 import jdk.vm.ci.meta.Assumptions; 133 import jdk.vm.ci.meta.ConstantReflectionProvider; 134 import jdk.vm.ci.meta.DeoptimizationAction; 135 import jdk.vm.ci.meta.DeoptimizationReason; 136 import jdk.vm.ci.meta.JavaConstant; 137 import jdk.vm.ci.meta.JavaKind; 138 import jdk.vm.ci.meta.JavaType; 139 import jdk.vm.ci.meta.MetaAccessProvider; 140 import jdk.vm.ci.meta.ResolvedJavaField; 141 import jdk.vm.ci.meta.ResolvedJavaMethod; 142 import jdk.vm.ci.meta.ResolvedJavaType; 143 144 /** 145 * A graph decoder that performs partial evaluation, i.e., that performs method inlining and 146 * canonicalization/simplification of nodes during decoding. 147 * 148 * Inlining and loop explosion are configured via the plugin mechanism also used by the 149 * {@link GraphBuilderPhase}. However, not all callback methods defined in 150 * {@link GraphBuilderContext} are available since decoding is more limited than graph building. 151 * 152 * The standard {@link Canonicalizable#canonical node canonicalization} interface is used to 153 * canonicalize nodes during decoding. Additionally, {@link IfNode branches} and 154 * {@link IntegerSwitchNode switches} with constant conditions are simplified. 155 */ 156 public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { 157 158 private static final Object CACHED_NULL_VALUE = new Object(); 159 160 public static class Options { 161 @Option(help = "Maximum inlining depth during partial evaluation before reporting an infinite recursion")// 162 public static final OptionKey<Integer> InliningDepthError = new OptionKey<>(1000); 163 164 @Option(help = "Max number of loop explosions per method.", type = OptionType.Debug)// 165 public static final OptionKey<Integer> MaximumLoopExplosionCount = new OptionKey<>(10000); 166 167 @Option(help = "Do not bail out but throw an exception on failed loop explosion.", type = OptionType.Debug)// 168 public static final OptionKey<Boolean> FailedLoopExplosionIsFatal = new OptionKey<>(false); 169 } 170 171 protected class PEMethodScope extends MethodScope { 172 /** The state of the caller method. Only non-null during method inlining. */ 173 protected final PEMethodScope caller; 174 protected final ResolvedJavaMethod method; 175 protected final InvokeData invokeData; 176 protected final int inliningDepth; 177 178 protected final ValueNode[] arguments; 179 private SourceLanguagePosition sourceLanguagePosition = UnresolvedSourceLanguagePosition.INSTANCE; 180 181 protected FrameState outerState; 182 protected FrameState exceptionState; 183 protected ExceptionPlaceholderNode exceptionPlaceholderNode; 184 protected NodeSourcePosition callerBytecodePosition; 185 PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, int inliningDepth, LoopExplosionPlugin loopExplosionPlugin, ValueNode[] arguments)186 protected PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, 187 int inliningDepth, LoopExplosionPlugin loopExplosionPlugin, ValueNode[] arguments) { 188 super(callerLoopScope, targetGraph, encodedGraph, loopExplosionKind(method, loopExplosionPlugin)); 189 190 this.caller = caller; 191 this.method = method; 192 this.invokeData = invokeData; 193 this.inliningDepth = inliningDepth; 194 this.arguments = arguments; 195 } 196 197 @Override isInlinedMethod()198 public boolean isInlinedMethod() { 199 return caller != null; 200 } 201 202 /** 203 * Gets the call stack representing this method scope and its callers. 204 */ getCallStack()205 public StackTraceElement[] getCallStack() { 206 StackTraceElement[] stack = new StackTraceElement[inliningDepth + 1]; 207 PEMethodScope frame = this; 208 int index = 0; 209 int bci = -1; 210 while (frame != null) { 211 stack[index++] = frame.method.asStackTraceElement(bci); 212 bci = frame.invokeData == null ? 0 : frame.invokeData.invoke.bci(); 213 frame = frame.caller; 214 } 215 assert index == stack.length : index + " != " + stack.length; 216 return stack; 217 } 218 219 @Override getCallerBytecodePosition(NodeSourcePosition position)220 public NodeSourcePosition getCallerBytecodePosition(NodeSourcePosition position) { 221 if (caller == null) { 222 return position; 223 } 224 if (callerBytecodePosition == null) { 225 NodeSourcePosition invokePosition = invokeData.invoke.asNode().getNodeSourcePosition(); 226 if (invokePosition == null) { 227 assert position == null : "should only happen when tracking is disabled"; 228 return null; 229 } 230 callerBytecodePosition = invokePosition; 231 } 232 if (position != null) { 233 return position.addCaller(caller.resolveSourceLanguagePosition(), callerBytecodePosition); 234 } 235 final SourceLanguagePosition pos = caller.resolveSourceLanguagePosition(); 236 if (pos != null && callerBytecodePosition != null) { 237 return new NodeSourcePosition(pos, callerBytecodePosition.getCaller(), callerBytecodePosition.getMethod(), callerBytecodePosition.getBCI()); 238 } 239 return callerBytecodePosition; 240 } 241 resolveSourceLanguagePosition()242 private SourceLanguagePosition resolveSourceLanguagePosition() { 243 SourceLanguagePosition res = sourceLanguagePosition; 244 if (res == UnresolvedSourceLanguagePosition.INSTANCE) { 245 res = null; 246 if (arguments != null && method.hasReceiver() && arguments.length > 0 && arguments[0].isJavaConstant()) { 247 JavaConstant constantArgument = arguments[0].asJavaConstant(); 248 res = sourceLanguagePositionProvider.getPosition(constantArgument); 249 } 250 sourceLanguagePosition = res; 251 } 252 return res; 253 } 254 255 @Override toString()256 public String toString() { 257 return getClass().getSimpleName() + '[' + method.format("%H.%n(%p)") + ']'; 258 } 259 } 260 261 private static final class UnresolvedSourceLanguagePosition implements SourceLanguagePosition { 262 static final SourceLanguagePosition INSTANCE = new UnresolvedSourceLanguagePosition(); 263 264 @Override toShortString()265 public String toShortString() { 266 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 267 } 268 269 @Override getOffsetEnd()270 public int getOffsetEnd() { 271 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 272 } 273 274 @Override getOffsetStart()275 public int getOffsetStart() { 276 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 277 } 278 279 @Override getLineNumber()280 public int getLineNumber() { 281 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 282 } 283 284 @Override getURI()285 public URI getURI() { 286 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 287 } 288 289 @Override getLanguage()290 public String getLanguage() { 291 throw new IllegalStateException(getClass().getSimpleName() + " should not be reachable."); 292 } 293 } 294 295 protected class PENonAppendGraphBuilderContext implements GraphBuilderContext { 296 protected final PEMethodScope methodScope; 297 protected final Invoke invoke; 298 299 @Override getExternalInliningContext()300 public ExternalInliningContext getExternalInliningContext() { 301 return new ExternalInliningContext() { 302 @Override 303 public int getInlinedDepth() { 304 int count = 0; 305 PEGraphDecoder.PEMethodScope scope = methodScope; 306 while (scope != null) { 307 if (scope.method.equals(peRootForInlining)) { 308 count++; 309 } 310 scope = scope.caller; 311 } 312 return count; 313 } 314 }; 315 } 316 PENonAppendGraphBuilderContext(PEMethodScope methodScope, Invoke invoke)317 public PENonAppendGraphBuilderContext(PEMethodScope methodScope, Invoke invoke) { 318 this.methodScope = methodScope; 319 this.invoke = invoke; 320 } 321 322 /** 323 * {@link Fold} and {@link NodeIntrinsic} can be deferred during parsing/decoding. Only by 324 * the end of {@linkplain SnippetTemplate#instantiate Snippet instantiation} do they need to 325 * have been processed. 326 * 327 * This is how SVM handles snippets. They are parsed with plugins disabled and then encoded 328 * and stored in the image. When the snippet is needed at runtime the graph is decoded and 329 * the plugins are run during the decoding process. If they aren't handled at this point 330 * then they will never be handled. 331 */ 332 @Override canDeferPlugin(GeneratedInvocationPlugin plugin)333 public boolean canDeferPlugin(GeneratedInvocationPlugin plugin) { 334 return plugin.isGeneratedFromFoldOrNodeIntrinsic(); 335 } 336 337 @Override bailout(String string)338 public BailoutException bailout(String string) { 339 BailoutException bailout = new PermanentBailoutException(string); 340 throw GraphUtil.createBailoutException(string, bailout, methodScope.getCallStack()); 341 } 342 343 @Override getStampProvider()344 public StampProvider getStampProvider() { 345 return providers.getStampProvider(); 346 } 347 348 @Override getMetaAccess()349 public MetaAccessProvider getMetaAccess() { 350 return providers.getMetaAccess(); 351 } 352 353 @Override getConstantReflection()354 public ConstantReflectionProvider getConstantReflection() { 355 return providers.getConstantReflection(); 356 } 357 358 @Override getConstantFieldProvider()359 public ConstantFieldProvider getConstantFieldProvider() { 360 return providers.getConstantFieldProvider(); 361 } 362 363 @Override getReplacements()364 public Replacements getReplacements() { 365 return providers.getReplacements(); 366 } 367 368 @Override getGraph()369 public StructuredGraph getGraph() { 370 return graph; 371 } 372 373 @Override getDepth()374 public int getDepth() { 375 return methodScope.inliningDepth; 376 } 377 378 @Override getIntrinsic()379 public IntrinsicContext getIntrinsic() { 380 return PEGraphDecoder.this.getIntrinsic(); 381 } 382 383 @Override append(T value)384 public <T extends ValueNode> T append(T value) { 385 throw unimplemented(); 386 } 387 388 @Override push(JavaKind kind, ValueNode value)389 public void push(JavaKind kind, ValueNode value) { 390 throw unimplemented(); 391 } 392 393 @Override handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything)394 public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { 395 throw unimplemented(); 396 } 397 398 @Override handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType)399 public void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType) { 400 throw unimplemented(); 401 } 402 403 @Override intrinsify(BytecodeProvider bytecodeProvider, ResolvedJavaMethod targetMethod, ResolvedJavaMethod substitute, InvocationPlugin.Receiver receiver, ValueNode[] args)404 public boolean intrinsify(BytecodeProvider bytecodeProvider, ResolvedJavaMethod targetMethod, ResolvedJavaMethod substitute, InvocationPlugin.Receiver receiver, ValueNode[] args) { 405 return false; 406 } 407 408 @Override intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver)409 public boolean intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver) { 410 return false; 411 } 412 413 @Override setStateAfter(StateSplit stateSplit)414 public void setStateAfter(StateSplit stateSplit) { 415 throw unimplemented(); 416 } 417 418 @Override getParent()419 public GraphBuilderContext getParent() { 420 throw unimplemented(); 421 } 422 423 @Override getCode()424 public Bytecode getCode() { 425 throw unimplemented(); 426 } 427 428 @Override getMethod()429 public ResolvedJavaMethod getMethod() { 430 return methodScope.method; 431 } 432 433 @Override bci()434 public int bci() { 435 // There is no BCI available when decoding an encoded method 436 return -1; 437 } 438 439 @Override getInvokeKind()440 public InvokeKind getInvokeKind() { 441 throw unimplemented(); 442 } 443 444 @Override getInvokeReturnType()445 public JavaType getInvokeReturnType() { 446 throw unimplemented(); 447 } 448 449 @Override toString()450 public String toString() { 451 Formatter fmt = new Formatter(); 452 fmt.format("Decoding %s", methodScope.method.format("%H.%n(%p)")); 453 for (StackTraceElement e : methodScope.getCallStack()) { 454 fmt.format("%n\tat %s", e); 455 } 456 return fmt.toString(); 457 } 458 } 459 460 protected IntrinsicContext getIntrinsic() { 461 return null; 462 } 463 464 protected class PEAppendGraphBuilderContext extends PENonAppendGraphBuilderContext { 465 protected FixedWithNextNode lastInstr; 466 protected ValueNode pushedNode; 467 protected boolean invokeConsumed; 468 protected final InvokeKind invokeKind; 469 protected final JavaType invokeReturnType; 470 471 public PEAppendGraphBuilderContext(PEMethodScope inlineScope, FixedWithNextNode lastInstr) { 472 this(inlineScope, lastInstr, null, null); 473 } 474 475 public PEAppendGraphBuilderContext(PEMethodScope inlineScope, FixedWithNextNode lastInstr, InvokeKind invokeKind, JavaType invokeReturnType) { 476 super(inlineScope, inlineScope.invokeData != null ? inlineScope.invokeData.invoke : null); 477 this.lastInstr = lastInstr; 478 this.invokeKind = invokeKind; 479 this.invokeReturnType = invokeReturnType; 480 } 481 482 @Override 483 public void push(JavaKind kind, ValueNode value) { 484 if (pushedNode != null) { 485 throw unimplemented("Only one push is supported"); 486 } 487 pushedNode = value; 488 } 489 490 @Override 491 public void setStateAfter(StateSplit stateSplit) { 492 Node stateAfter = decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); 493 getGraph().add(stateAfter); 494 FrameState fs = (FrameState) handleFloatingNodeAfterAdd(methodScope.caller, methodScope.callerLoopScope, stateAfter); 495 stateSplit.setStateAfter(fs); 496 } 497 498 @SuppressWarnings("try") 499 @Override 500 public <T extends ValueNode> T append(T v) { 501 if (v.graph() != null) { 502 return v; 503 } 504 try (DebugCloseable position = withNodeSoucePosition()) { 505 T added = getGraph().addOrUniqueWithInputs(v); 506 if (added == v) { 507 updateLastInstruction(v); 508 } 509 return added; 510 } 511 } 512 513 private DebugCloseable withNodeSoucePosition() { 514 if (getGraph().trackNodeSourcePosition()) { 515 NodeSourcePosition callerBytecodePosition = methodScope.getCallerBytecodePosition(); 516 if (callerBytecodePosition != null) { 517 return getGraph().withNodeSourcePosition(callerBytecodePosition); 518 } 519 } 520 return null; 521 } 522 523 private <T extends ValueNode> void updateLastInstruction(T v) { 524 if (v instanceof FixedNode) { 525 FixedNode fixedNode = (FixedNode) v; 526 if (lastInstr != null) { 527 lastInstr.setNext(fixedNode); 528 } 529 if (fixedNode instanceof FixedWithNextNode) { 530 FixedWithNextNode fixedWithNextNode = (FixedWithNextNode) fixedNode; 531 assert fixedWithNextNode.next() == null : "cannot append instruction to instruction which isn't end"; 532 lastInstr = fixedWithNextNode; 533 } else { 534 lastInstr = null; 535 } 536 } 537 } 538 539 @Override 540 public InvokeKind getInvokeKind() { 541 if (invokeKind != null) { 542 return invokeKind; 543 } 544 return super.getInvokeKind(); 545 } 546 547 @Override 548 public JavaType getInvokeReturnType() { 549 if (invokeReturnType != null) { 550 return invokeReturnType; 551 } 552 return super.getInvokeReturnType(); 553 } 554 555 @Override 556 public void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType) { 557 if (invokeConsumed) { 558 throw unimplemented("handleReplacedInvoke can be called only once"); 559 } 560 invokeConsumed = true; 561 562 appendInvoke(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData, callTarget); 563 updateLastInstruction(invoke.asNode()); 564 } 565 566 @Override 567 public GraphBuilderContext getNonIntrinsicAncestor() { 568 return null; 569 } 570 } 571 572 protected class PEPluginGraphBuilderContext extends PENonAppendGraphBuilderContext { 573 protected FixedWithNextNode insertBefore; 574 protected ValueNode pushedNode; 575 576 public PEPluginGraphBuilderContext(PEMethodScope inlineScope, FixedWithNextNode insertBefore) { 577 super(inlineScope, inlineScope.invokeData != null ? inlineScope.invokeData.invoke : null); 578 this.insertBefore = insertBefore; 579 } 580 581 @Override 582 public void push(JavaKind kind, ValueNode value) { 583 if (pushedNode != null) { 584 throw unimplemented("Only one push is supported"); 585 } 586 pushedNode = value; 587 } 588 589 @Override 590 public void setStateAfter(StateSplit sideEffect) { 591 assert sideEffect.hasSideEffect(); 592 FrameState stateAfter = getGraph().add(new FrameState(BytecodeFrame.BEFORE_BCI)); 593 sideEffect.setStateAfter(stateAfter); 594 } 595 596 @SuppressWarnings("try") 597 @Override 598 public <T extends ValueNode> T append(T v) { 599 if (v.graph() != null) { 600 return v; 601 } 602 try (DebugCloseable position = withNodeSoucePosition()) { 603 T added = getGraph().addOrUniqueWithInputs(v); 604 if (added == v) { 605 updateLastInstruction(v); 606 } 607 return added; 608 } 609 } 610 611 private DebugCloseable withNodeSoucePosition() { 612 if (getGraph().trackNodeSourcePosition()) { 613 NodeSourcePosition callerBytecodePosition = methodScope.getCallerBytecodePosition(); 614 if (callerBytecodePosition != null) { 615 return getGraph().withNodeSourcePosition(callerBytecodePosition); 616 } 617 } 618 return null; 619 } 620 621 private <T extends ValueNode> void updateLastInstruction(T value) { 622 if (value instanceof FixedWithNextNode) { 623 FixedWithNextNode fixed = (FixedWithNextNode) value; 624 graph.addBeforeFixed(insertBefore, fixed); 625 } else if (value instanceof FixedNode) { 626 // Block terminating fixed nodes shouldn't be inserted 627 throw GraalError.shouldNotReachHere(); 628 } 629 } 630 } 631 632 @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED, allowedUsageTypes = {InputType.Value, InputType.Guard, InputType.Anchor}) 633 static class ExceptionPlaceholderNode extends ValueNode { 634 public static final NodeClass<ExceptionPlaceholderNode> TYPE = NodeClass.create(ExceptionPlaceholderNode.class); 635 636 protected ExceptionPlaceholderNode() { 637 super(TYPE, StampFactory.object()); 638 } 639 } 640 641 @NodeInfo(allowedUsageTypes = {Anchor, Guard, InputType.Value}, cycles = CYCLES_0, size = SIZE_0) 642 static final class FixedAnchorNode extends FixedWithNextNode implements AnchoringNode, GuardingNode, IterableNodeType { 643 644 public static final NodeClass<FixedAnchorNode> TYPE = NodeClass.create(FixedAnchorNode.class); 645 @Input ValueNode value; 646 647 protected FixedAnchorNode(ValueNode value) { 648 super(TYPE, value.stamp(NodeView.DEFAULT)); 649 this.value = value; 650 } 651 } 652 653 protected static class SpecialCallTargetCacheKey { 654 private final InvokeKind invokeKind; 655 private final ResolvedJavaMethod targetMethod; 656 private final ResolvedJavaType contextType; 657 private final Stamp receiverStamp; 658 659 public SpecialCallTargetCacheKey(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ResolvedJavaType contextType, Stamp receiverStamp) { 660 this.invokeKind = invokeKind; 661 this.targetMethod = targetMethod; 662 this.contextType = contextType; 663 this.receiverStamp = receiverStamp; 664 } 665 666 @Override 667 public int hashCode() { 668 return invokeKind.hashCode() ^ targetMethod.hashCode() ^ contextType.hashCode() ^ receiverStamp.hashCode(); 669 } 670 671 @Override 672 public boolean equals(Object obj) { 673 if (obj instanceof SpecialCallTargetCacheKey) { 674 SpecialCallTargetCacheKey key = (SpecialCallTargetCacheKey) obj; 675 return key.invokeKind.equals(this.invokeKind) && key.targetMethod.equals(this.targetMethod) && key.contextType.equals(this.contextType) && key.receiverStamp.equals(this.receiverStamp); 676 } 677 return false; 678 } 679 } 680 681 private final LoopExplosionPlugin loopExplosionPlugin; 682 private final InvocationPlugins invocationPlugins; 683 private final InlineInvokePlugin[] inlineInvokePlugins; 684 private final ParameterPlugin parameterPlugin; 685 private final NodePlugin[] nodePlugins; 686 private final EconomicMap<SpecialCallTargetCacheKey, Object> specialCallTargetCache; 687 private final EconomicMap<ResolvedJavaMethod, Object> invocationPluginCache; 688 private final ResolvedJavaMethod peRootForInlining; 689 protected final SourceLanguagePositionProvider sourceLanguagePositionProvider; 690 691 public PEGraphDecoder(Architecture architecture, StructuredGraph graph, CoreProviders providers, LoopExplosionPlugin loopExplosionPlugin, InvocationPlugins invocationPlugins, 692 InlineInvokePlugin[] inlineInvokePlugins, 693 ParameterPlugin parameterPlugin, 694 NodePlugin[] nodePlugins, ResolvedJavaMethod peRootForInlining, SourceLanguagePositionProvider sourceLanguagePositionProvider) { 695 super(architecture, graph, providers, true); 696 this.loopExplosionPlugin = loopExplosionPlugin; 697 this.invocationPlugins = invocationPlugins; 698 this.inlineInvokePlugins = inlineInvokePlugins; 699 this.parameterPlugin = parameterPlugin; 700 this.nodePlugins = nodePlugins; 701 this.specialCallTargetCache = EconomicMap.create(Equivalence.DEFAULT); 702 this.invocationPluginCache = EconomicMap.create(Equivalence.DEFAULT); 703 this.peRootForInlining = peRootForInlining; 704 this.sourceLanguagePositionProvider = sourceLanguagePositionProvider; 705 } 706 707 protected static LoopExplosionKind loopExplosionKind(ResolvedJavaMethod method, LoopExplosionPlugin loopExplosionPlugin) { 708 if (loopExplosionPlugin == null) { 709 return LoopExplosionKind.NONE; 710 } else { 711 return loopExplosionPlugin.loopExplosionKind(method); 712 } 713 } 714 715 @SuppressWarnings("try") 716 public void decode(ResolvedJavaMethod method, boolean isSubstitution, boolean trackNodeSourcePosition) { 717 try (DebugContext.Scope scope = debug.scope("PEGraphDecode", graph)) { 718 EncodedGraph encodedGraph = lookupEncodedGraph(method, null, null, isSubstitution, trackNodeSourcePosition); 719 recordGraphElements(encodedGraph); 720 PEMethodScope methodScope = new PEMethodScope(graph, null, null, encodedGraph, method, null, 0, loopExplosionPlugin, null); 721 decode(createInitialLoopScope(methodScope, null)); 722 cleanupGraph(methodScope); 723 724 debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After graph cleanup"); 725 assert graph.verify(); 726 } catch (Throwable t) { 727 throw debug.handle(t); 728 } 729 730 try { 731 /* Check that the control flow graph can be computed, to catch problems early. */ 732 assert CFGVerifier.verify(ControlFlowGraph.compute(graph, true, true, true, true)); 733 } catch (Throwable ex) { 734 throw GraalError.shouldNotReachHere(ex, "Control flow graph not valid after partial evaluation"); 735 } 736 } 737 738 private void recordGraphElements(EncodedGraph encodedGraph) { 739 List<ResolvedJavaMethod> inlinedMethods = encodedGraph.getInlinedMethods(); 740 if (inlinedMethods != null) { 741 for (ResolvedJavaMethod other : inlinedMethods) { 742 graph.recordMethod(other); 743 } 744 } 745 Assumptions assumptions = graph.getAssumptions(); 746 Assumptions inlinedAssumptions = encodedGraph.getAssumptions(); 747 if (assumptions != null) { 748 if (inlinedAssumptions != null) { 749 assumptions.record(inlinedAssumptions); 750 } 751 } else { 752 assert inlinedAssumptions == null : String.format("cannot inline graph (%s) which makes assumptions into a graph (%s) that doesn't", encodedGraph, graph); 753 } 754 if (encodedGraph.getFields() != null) { 755 for (ResolvedJavaField field : encodedGraph.getFields()) { 756 graph.recordField(field); 757 } 758 } 759 if (encodedGraph.hasUnsafeAccess()) { 760 graph.markUnsafeAccess(); 761 } 762 } 763 764 @Override 765 protected void cleanupGraph(MethodScope methodScope) { 766 super.cleanupGraph(methodScope); 767 768 for (FrameState frameState : graph.getNodes(FrameState.TYPE)) { 769 if (frameState.bci == BytecodeFrame.UNWIND_BCI) { 770 /* 771 * handleMissingAfterExceptionFrameState is called during graph decoding from 772 * InliningUtil.processFrameState - but during graph decoding it does not do 773 * anything because the usages of the frameState are not available yet. So we need 774 * to call it again. 775 */ 776 PEMethodScope peMethodScope = (PEMethodScope) methodScope; 777 Invoke invoke = peMethodScope.invokeData != null ? peMethodScope.invokeData.invoke : null; 778 InliningUtil.handleMissingAfterExceptionFrameState(frameState, invoke, null, true); 779 780 /* 781 * The frameState must be gone now, because it is not a valid deoptimization point. 782 */ 783 assert frameState.isDeleted(); 784 } 785 } 786 /* 787 * Cleanup anchor nodes introduced for exception object anchors during inlining. When we 788 * inline through an invoke with exception and the caller has an exception handler attached 789 * this handler can use the exception object node as anchor or guard. During inlining the 790 * exception object node is replaced with its actual value produced in the callee, however 791 * this value is not necessarily a guarding or anchoring node. Thus, we introduce artificial 792 * fixed anchor nodes in the exceptional path in the callee the caller can use as anchor, 793 * guard or value input. After we are done we remove all anchors that are not needed any 794 * more, i.e., they do not have any guard or anchor usages. If they have guard or anchor 795 * usages we rewrite the anchor for exactly those nodes to a real value anchor node that can 796 * be optimized later. 797 * 798 * Optimizing the anchor early is not possible during partial evaluation, since we do not 799 * know if a node not yet decoded in the caller will reference the exception object 800 * replacement node in the callee as an anchor or guard. 801 */ 802 for (FixedAnchorNode anchor : graph.getNodes(FixedAnchorNode.TYPE).snapshot()) { 803 AbstractBeginNode newAnchor = AbstractBeginNode.prevBegin(anchor); 804 assert newAnchor != null : "Must find prev begin node"; 805 anchor.replaceAtUsages(newAnchor, InputType.Guard, InputType.Anchor); 806 // all other usages can really consume the value 807 anchor.replaceAtUsages(anchor.value); 808 assert anchor.hasNoUsages(); 809 GraphUtil.unlinkFixedNode(anchor); 810 anchor.safeDelete(); 811 } 812 } 813 814 @Override 815 protected void checkLoopExplosionIteration(MethodScope s, LoopScope loopScope) { 816 PEMethodScope methodScope = (PEMethodScope) s; 817 if (loopScope.loopIteration > Options.MaximumLoopExplosionCount.getValue(options)) { 818 throw tooManyLoopExplosionIterations(methodScope, options); 819 } 820 } 821 822 private static RuntimeException tooManyLoopExplosionIterations(PEMethodScope methodScope, OptionValues options) { 823 String message = "too many loop explosion iterations - does the explosion not terminate for method " + methodScope.method + "?"; 824 RuntimeException bailout = Options.FailedLoopExplosionIsFatal.getValue(options) ? new RuntimeException(message) : new PermanentBailoutException(message); 825 throw GraphUtil.createBailoutException(message, bailout, methodScope.getCallStack()); 826 } 827 828 @Override 829 protected LoopScope handleInvoke(MethodScope s, LoopScope loopScope, InvokeData invokeData) { 830 PEMethodScope methodScope = (PEMethodScope) s; 831 /* 832 * Decode the call target, but do not add it to the graph yet. This avoids adding usages for 833 * all the arguments, which are expensive to remove again when we can inline the method. 834 */ 835 assert invokeData.invoke.callTarget() == null : "callTarget edge is ignored during decoding of Invoke"; 836 CallTargetNode callTarget = (CallTargetNode) decodeFloatingNode(methodScope, loopScope, invokeData.callTargetOrderId); 837 if (callTarget instanceof MethodCallTargetNode) { 838 MethodCallTargetNode methodCall = (MethodCallTargetNode) callTarget; 839 if (methodCall.invokeKind().hasReceiver()) { 840 invokeData.constantReceiver = methodCall.arguments().get(0).asJavaConstant(); 841 } 842 callTarget = trySimplifyCallTarget(methodScope, invokeData, (MethodCallTargetNode) callTarget); 843 LoopScope inlineLoopScope = trySimplifyInvoke(methodScope, loopScope, invokeData, (MethodCallTargetNode) callTarget); 844 if (inlineLoopScope != null) { 845 return inlineLoopScope; 846 } 847 } 848 849 /* We know that we need an invoke, so now we can add the call target to the graph. */ 850 graph.add(callTarget); 851 registerNode(loopScope, invokeData.callTargetOrderId, callTarget, false, false); 852 return super.handleInvoke(methodScope, loopScope, invokeData); 853 } 854 855 protected MethodCallTargetNode trySimplifyCallTarget(PEMethodScope methodScope, InvokeData invokeData, MethodCallTargetNode callTarget) { 856 // attempt to devirtualize the call 857 ResolvedJavaMethod specialCallTarget = getSpecialCallTarget(invokeData, callTarget); 858 if (specialCallTarget != null) { 859 callTarget.setTargetMethod(specialCallTarget); 860 callTarget.setInvokeKind(InvokeKind.Special); 861 return callTarget; 862 } 863 if (callTarget.invokeKind().isInterface()) { 864 Invoke invoke = invokeData.invoke; 865 ResolvedJavaType contextType = methodScope.method.getDeclaringClass(); 866 return MethodCallTargetNode.tryDevirtualizeInterfaceCall(callTarget.receiver(), callTarget.targetMethod(), null, graph.getAssumptions(), contextType, callTarget, invoke.asNode()); 867 } 868 return callTarget; 869 } 870 871 protected LoopScope trySimplifyInvoke(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { 872 if (tryInvocationPlugin(methodScope, loopScope, invokeData, callTarget)) { 873 /* 874 * The invocation plugin handled the call, so decoding continues in the calling method. 875 */ 876 return loopScope; 877 } 878 LoopScope inlineLoopScope = tryInline(methodScope, loopScope, invokeData, callTarget); 879 if (inlineLoopScope != null) { 880 /* 881 * We can inline the call, so decoding continues in the inlined method. 882 */ 883 return inlineLoopScope; 884 } 885 886 for (InlineInvokePlugin plugin : inlineInvokePlugins) { 887 plugin.notifyNotInlined(new PENonAppendGraphBuilderContext(methodScope, invokeData.invoke), callTarget.targetMethod(), invokeData.invoke); 888 } 889 return null; 890 } 891 892 private ResolvedJavaMethod getSpecialCallTarget(InvokeData invokeData, MethodCallTargetNode callTarget) { 893 if (callTarget.invokeKind().isDirect()) { 894 return null; 895 } 896 897 // check for trivial cases (e.g. final methods, nonvirtual methods) 898 if (callTarget.targetMethod().canBeStaticallyBound()) { 899 return callTarget.targetMethod(); 900 } 901 902 SpecialCallTargetCacheKey key = new SpecialCallTargetCacheKey(callTarget.invokeKind(), callTarget.targetMethod(), invokeData.contextType, callTarget.receiver().stamp(NodeView.DEFAULT)); 903 Object specialCallTarget = specialCallTargetCache.get(key); 904 if (specialCallTarget == null) { 905 specialCallTarget = MethodCallTargetNode.devirtualizeCall(key.invokeKind, key.targetMethod, key.contextType, graph.getAssumptions(), 906 key.receiverStamp); 907 if (specialCallTarget == null) { 908 specialCallTarget = CACHED_NULL_VALUE; 909 } 910 specialCallTargetCache.put(key, specialCallTarget); 911 } 912 913 return specialCallTarget == CACHED_NULL_VALUE ? null : (ResolvedJavaMethod) specialCallTarget; 914 } 915 916 protected boolean tryInvocationPlugin(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { 917 if (invocationPlugins == null || invocationPlugins.isEmpty()) { 918 return false; 919 } 920 921 Invoke invoke = invokeData.invoke; 922 923 ResolvedJavaMethod targetMethod = callTarget.targetMethod(); 924 if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { 925 return false; 926 } 927 928 InvocationPlugin invocationPlugin = getInvocationPlugin(targetMethod); 929 if (invocationPlugin == null) { 930 return false; 931 } 932 933 if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { 934 return false; 935 } 936 937 ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); 938 FixedWithNextNode invokePredecessor = (FixedWithNextNode) invoke.asNode().predecessor(); 939 940 /* 941 * Remove invoke from graph so that invocation plugin can append nodes to the predecessor. 942 */ 943 invoke.asNode().replaceAtPredecessor(null); 944 945 PEMethodScope inlineScope = new PEMethodScope(graph, methodScope, loopScope, null, targetMethod, invokeData, methodScope.inliningDepth + 1, loopExplosionPlugin, arguments); 946 947 JavaType returnType = targetMethod.getSignature().getReturnType(methodScope.method.getDeclaringClass()); 948 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(inlineScope, invokePredecessor, callTarget.invokeKind(), returnType); 949 InvocationPluginReceiver invocationPluginReceiver = new InvocationPluginReceiver(graphBuilderContext); 950 951 if (invocationPlugin.execute(graphBuilderContext, targetMethod, invocationPluginReceiver.init(targetMethod, arguments), arguments)) { 952 953 if (graphBuilderContext.invokeConsumed) { 954 /* Nothing to do. */ 955 } else if (graphBuilderContext.lastInstr != null) { 956 if (graphBuilderContext.lastInstr instanceof DeoptBciSupplier && !BytecodeFrame.isPlaceholderBci(invokeData.invoke.bci()) && 957 BytecodeFrame.isPlaceholderBci(((DeoptBciSupplier) graphBuilderContext.lastInstr).bci())) { 958 ((DeoptBciSupplier) graphBuilderContext.lastInstr).setBci(invokeData.invoke.bci()); 959 } 960 registerNode(loopScope, invokeData.invokeOrderId, graphBuilderContext.pushedNode, true, true); 961 invoke.asNode().replaceAtUsages(graphBuilderContext.pushedNode); 962 graphBuilderContext.lastInstr.setNext(nodeAfterInvoke(methodScope, loopScope, invokeData, AbstractBeginNode.prevBegin(graphBuilderContext.lastInstr))); 963 deleteInvoke(invoke); 964 } else { 965 assert graphBuilderContext.pushedNode == null : "Why push a node when the invoke does not return anyway?"; 966 invoke.asNode().replaceAtUsages(null); 967 deleteInvoke(invoke); 968 } 969 return true; 970 971 } else { 972 /* Intrinsification failed, restore original state: invoke is in Graph. */ 973 invokePredecessor.setNext(invoke.asNode()); 974 return false; 975 } 976 } 977 978 private InvocationPlugin getInvocationPlugin(ResolvedJavaMethod targetMethod) { 979 Object invocationPlugin = invocationPluginCache.get(targetMethod); 980 if (invocationPlugin == null) { 981 invocationPlugin = invocationPlugins.lookupInvocation(targetMethod); 982 if (invocationPlugin == null) { 983 invocationPlugin = CACHED_NULL_VALUE; 984 } 985 invocationPluginCache.put(targetMethod, invocationPlugin); 986 } 987 988 return invocationPlugin == CACHED_NULL_VALUE ? null : (InvocationPlugin) invocationPlugin; 989 } 990 991 protected LoopScope tryInline(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { 992 if (!callTarget.invokeKind().isDirect()) { 993 return null; 994 } 995 996 ResolvedJavaMethod targetMethod = callTarget.targetMethod(); 997 if (targetMethod.hasNeverInlineDirective()) { 998 return null; 999 } 1000 1001 ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); 1002 GraphBuilderContext graphBuilderContext = new PENonAppendGraphBuilderContext(methodScope, invokeData.invoke); 1003 1004 for (InlineInvokePlugin plugin : inlineInvokePlugins) { 1005 InlineInfo inlineInfo = plugin.shouldInlineInvoke(graphBuilderContext, targetMethod, arguments); 1006 if (inlineInfo != null) { 1007 if (inlineInfo.allowsInlining()) { 1008 return doInline(methodScope, loopScope, invokeData, inlineInfo, arguments); 1009 } else { 1010 return null; 1011 } 1012 } 1013 } 1014 return null; 1015 } 1016 1017 protected LoopScope doInline(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, InlineInfo inlineInfo, ValueNode[] arguments) { 1018 if (!invokeData.invoke.useForInlining()) { 1019 return null; 1020 } 1021 ResolvedJavaMethod inlineMethod = inlineInfo.getMethodToInline(); 1022 EncodedGraph graphToInline = lookupEncodedGraph(inlineMethod, inlineInfo.getPlugin(), inlineInfo.getIntrinsicBytecodeProvider(), inlineInfo.isSubstitution(), graph.trackNodeSourcePosition()); 1023 if (graphToInline == null) { 1024 return null; 1025 } 1026 1027 assert !graph.trackNodeSourcePosition() || graphToInline.trackNodeSourcePosition() : graph + " " + graphToInline; 1028 if (methodScope.inliningDepth > Options.InliningDepthError.getValue(options)) { 1029 throw tooDeepInlining(methodScope); 1030 } 1031 1032 for (InlineInvokePlugin plugin : inlineInvokePlugins) { 1033 plugin.notifyBeforeInline(inlineMethod); 1034 } 1035 1036 Invoke invoke = invokeData.invoke; 1037 FixedNode invokeNode = invoke.asNode(); 1038 FixedWithNextNode predecessor = (FixedWithNextNode) invokeNode.predecessor(); 1039 invokeNode.replaceAtPredecessor(null); 1040 1041 PEMethodScope inlineScope = new PEMethodScope(graph, methodScope, loopScope, graphToInline, inlineMethod, invokeData, methodScope.inliningDepth + 1, 1042 loopExplosionPlugin, arguments); 1043 1044 if (!inlineMethod.isStatic()) { 1045 if (StampTool.isPointerAlwaysNull(arguments[0])) { 1046 /* 1047 * The receiver is null, so we can unconditionally throw a NullPointerException 1048 * instead of performing any inlining. 1049 */ 1050 DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException)); 1051 predecessor.setNext(deoptimizeNode); 1052 finishInlining(inlineScope); 1053 /* Continue decoding in the caller. */ 1054 return loopScope; 1055 1056 } else if (!StampTool.isPointerNonNull(arguments[0])) { 1057 /* The receiver might be null, so we need to insert a null check. */ 1058 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(inlineScope, predecessor); 1059 arguments[0] = graphBuilderContext.nullCheckedValue(arguments[0]); 1060 predecessor = graphBuilderContext.lastInstr; 1061 } 1062 } 1063 1064 LoopScope inlineLoopScope = createInitialLoopScope(inlineScope, predecessor); 1065 1066 /* 1067 * The GraphEncoder assigns parameters a nodeId immediately after the fixed nodes. 1068 * Initializing createdNodes here avoid decoding and immediately replacing the 1069 * ParameterNodes. 1070 */ 1071 int firstArgumentNodeId = inlineScope.maxFixedNodeOrderId + 1; 1072 for (int i = 0; i < arguments.length; i++) { 1073 inlineLoopScope.createdNodes[firstArgumentNodeId + i] = arguments[i]; 1074 } 1075 1076 // Copy inlined methods from inlinee to caller 1077 recordGraphElements(graphToInline); 1078 1079 /* 1080 * Do the actual inlining by returning the initial loop scope for the inlined method scope. 1081 */ 1082 return inlineLoopScope; 1083 } 1084 1085 @Override 1086 protected void finishInlining(MethodScope is) { 1087 PEMethodScope inlineScope = (PEMethodScope) is; 1088 ResolvedJavaMethod inlineMethod = inlineScope.method; 1089 PEMethodScope methodScope = inlineScope.caller; 1090 LoopScope loopScope = inlineScope.callerLoopScope; 1091 InvokeData invokeData = inlineScope.invokeData; 1092 Invoke invoke = invokeData.invoke; 1093 FixedNode invokeNode = invoke.asNode(); 1094 1095 ValueNode exceptionValue = null; 1096 int returnNodeCount = 0; 1097 int unwindNodeCount = 0; 1098 List<ControlSinkNode> returnAndUnwindNodes = inlineScope.returnAndUnwindNodes; 1099 for (int i = 0; i < returnAndUnwindNodes.size(); i++) { 1100 FixedNode fixedNode = returnAndUnwindNodes.get(i); 1101 if (fixedNode instanceof ReturnNode) { 1102 returnNodeCount++; 1103 } else if (fixedNode.isAlive()) { 1104 assert fixedNode instanceof UnwindNode; 1105 unwindNodeCount++; 1106 } 1107 } 1108 1109 if (unwindNodeCount > 0) { 1110 FixedNode unwindReplacement; 1111 if (invoke instanceof InvokeWithExceptionNode) { 1112 /* Decoding continues for the exception handler. */ 1113 unwindReplacement = makeStubNode(methodScope, loopScope, invokeData.exceptionNextOrderId); 1114 } else { 1115 /* No exception handler available, so the only thing we can do is deoptimize. */ 1116 unwindReplacement = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler)); 1117 } 1118 1119 if (unwindNodeCount == 1) { 1120 /* Only one UnwindNode, we can use the exception directly. */ 1121 UnwindNode unwindNode = getSingleMatchingNode(returnAndUnwindNodes, returnNodeCount > 0, UnwindNode.class); 1122 exceptionValue = unwindNode.exception(); 1123 unwindNode.replaceAndDelete(unwindReplacement); 1124 1125 } else { 1126 /* 1127 * More than one UnwindNode. This can happen with the loop explosion strategy 1128 * FULL_EXPLODE_UNTIL_RETURN, where we keep exploding after the loop and therefore 1129 * also explode exception paths. Merge the exception in a similar way as multiple 1130 * return values. 1131 */ 1132 MergeNode unwindMergeNode = graph.add(new MergeNode()); 1133 exceptionValue = InliningUtil.mergeValueProducers(unwindMergeNode, getMatchingNodes(returnAndUnwindNodes, returnNodeCount > 0, UnwindNode.class, unwindNodeCount), 1134 null, unwindNode -> unwindNode.exception()); 1135 unwindMergeNode.setNext(unwindReplacement); 1136 ensureExceptionStateDecoded(inlineScope); 1137 unwindMergeNode.setStateAfter(inlineScope.exceptionState.duplicateModified(JavaKind.Object, JavaKind.Object, exceptionValue)); 1138 } 1139 if (invoke instanceof InvokeWithExceptionNode) { 1140 /* 1141 * Exceptionobject nodes are begin nodes, i.e., they can be used as guards/anchors 1142 * thus we need to ensure nodes decoded later have a correct guarding/anchoring node 1143 * in place, i.e., the exception value must be a node that can be used like the 1144 * original node as value, guard and anchor. 1145 * 1146 * The node unwindReplacement is a stub, its not yet processed thus we insert our 1147 * artificial anchor before it. 1148 */ 1149 assert unwindReplacement != exceptionValue : "Unschedulable unwind replacement"; 1150 FixedAnchorNode anchor = graph.add(new FixedAnchorNode(exceptionValue)); 1151 graph.addBeforeFixed(unwindReplacement, anchor); 1152 exceptionValue = anchor; 1153 assert anchor.predecessor() != null; 1154 } 1155 } 1156 1157 assert invoke.next() == null; 1158 assert !(invoke instanceof InvokeWithExceptionNode) || ((InvokeWithExceptionNode) invoke).exceptionEdge() == null; 1159 1160 ValueNode returnValue; 1161 if (returnNodeCount == 0) { 1162 returnValue = null; 1163 } else if (returnNodeCount == 1) { 1164 ReturnNode returnNode = getSingleMatchingNode(returnAndUnwindNodes, unwindNodeCount > 0, ReturnNode.class); 1165 returnValue = returnNode.result(); 1166 FixedNode n = nodeAfterInvoke(methodScope, loopScope, invokeData, AbstractBeginNode.prevBegin(returnNode)); 1167 returnNode.replaceAndDelete(n); 1168 } else { 1169 AbstractMergeNode merge = graph.add(new MergeNode()); 1170 merge.setStateAfter((FrameState) ensureNodeCreated(methodScope, loopScope, invokeData.stateAfterOrderId)); 1171 returnValue = InliningUtil.mergeReturns(merge, getMatchingNodes(returnAndUnwindNodes, unwindNodeCount > 0, ReturnNode.class, returnNodeCount)); 1172 FixedNode n = nodeAfterInvoke(methodScope, loopScope, invokeData, merge); 1173 merge.setNext(n); 1174 } 1175 invokeNode.replaceAtUsages(returnValue); 1176 1177 /* 1178 * Usage the handles that we have on the return value and the exception to update the 1179 * orderId->Node table. 1180 */ 1181 registerNode(loopScope, invokeData.invokeOrderId, returnValue, true, true); 1182 if (invoke instanceof InvokeWithExceptionNode) { 1183 registerNode(loopScope, invokeData.exceptionOrderId, exceptionValue, true, true); 1184 } 1185 if (inlineScope.exceptionPlaceholderNode != null) { 1186 inlineScope.exceptionPlaceholderNode.replaceAtUsagesAndDelete(exceptionValue); 1187 } 1188 deleteInvoke(invoke); 1189 1190 assert exceptionValue == null || exceptionValue instanceof FixedAnchorNode && exceptionValue.predecessor() != null; 1191 1192 for (InlineInvokePlugin plugin : inlineInvokePlugins) { 1193 plugin.notifyAfterInline(inlineMethod); 1194 } 1195 } 1196 1197 @SuppressWarnings("unchecked") 1198 private static <T> T getSingleMatchingNode(List<ControlSinkNode> returnAndUnwindNodes, boolean hasNonMatchingEntries, Class<T> clazz) { 1199 if (!hasNonMatchingEntries) { 1200 assert returnAndUnwindNodes.size() == 1; 1201 return (T) returnAndUnwindNodes.get(0); 1202 } 1203 1204 for (int i = 0; i < returnAndUnwindNodes.size(); i++) { 1205 ControlSinkNode node = returnAndUnwindNodes.get(i); 1206 if (clazz.isInstance(node)) { 1207 return (T) node; 1208 } 1209 } 1210 throw GraalError.shouldNotReachHere(); 1211 } 1212 1213 @SuppressWarnings("unchecked") 1214 private static <T> List<T> getMatchingNodes(List<ControlSinkNode> returnAndUnwindNodes, boolean hasNonMatchingEntries, Class<T> clazz, int resultCount) { 1215 if (!hasNonMatchingEntries) { 1216 return (List<T>) returnAndUnwindNodes; 1217 } 1218 1219 List<T> result = new ArrayList<>(resultCount); 1220 for (int i = 0; i < returnAndUnwindNodes.size(); i++) { 1221 ControlSinkNode node = returnAndUnwindNodes.get(i); 1222 if (clazz.isInstance(node)) { 1223 result.add((T) node); 1224 } 1225 } 1226 assert result.size() == resultCount; 1227 return result; 1228 } 1229 1230 private static RuntimeException tooDeepInlining(PEMethodScope methodScope) { 1231 HashMap<ResolvedJavaMethod, Integer> methodCounts = new HashMap<>(); 1232 for (PEMethodScope cur = methodScope; cur != null; cur = cur.caller) { 1233 Integer oldCount = methodCounts.get(cur.method); 1234 methodCounts.put(cur.method, oldCount == null ? 1 : oldCount + 1); 1235 } 1236 1237 List<Map.Entry<ResolvedJavaMethod, Integer>> methods = new ArrayList<>(methodCounts.entrySet()); 1238 methods.sort((e1, e2) -> -Integer.compare(e1.getValue(), e2.getValue())); 1239 1240 StringBuilder msg = new StringBuilder("Too deep inlining, probably caused by recursive inlining.").append(System.lineSeparator()).append("== Inlined methods ordered by inlining frequency:"); 1241 for (Map.Entry<ResolvedJavaMethod, Integer> entry : methods) { 1242 msg.append(System.lineSeparator()).append(entry.getKey().format("%H.%n(%p) [")).append(entry.getValue()).append("]"); 1243 } 1244 msg.append(System.lineSeparator()).append("== Complete stack trace of inlined methods:"); 1245 int lastBci = 0; 1246 for (PEMethodScope cur = methodScope; cur != null; cur = cur.caller) { 1247 msg.append(System.lineSeparator()).append(cur.method.asStackTraceElement(lastBci)); 1248 if (cur.invokeData != null) { 1249 lastBci = cur.invokeData.invoke.bci(); 1250 } else { 1251 lastBci = 0; 1252 } 1253 } 1254 1255 throw new PermanentBailoutException(msg.toString()); 1256 } 1257 1258 public FixedNode nodeAfterInvoke(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, AbstractBeginNode lastBlock) { 1259 assert lastBlock.isAlive(); 1260 FixedNode n; 1261 if (invokeData.invoke instanceof InvokeWithExceptionNode) { 1262 registerNode(loopScope, invokeData.nextOrderId, lastBlock, false, false); 1263 n = makeStubNode(methodScope, loopScope, invokeData.nextNextOrderId); 1264 } else { 1265 n = makeStubNode(methodScope, loopScope, invokeData.nextOrderId); 1266 } 1267 return n; 1268 } 1269 1270 private static void deleteInvoke(Invoke invoke) { 1271 /* 1272 * Clean up unused nodes. We cannot just call killCFG on the invoke node because that can 1273 * kill too much: nodes that are decoded later can use values that appear unused by now. 1274 */ 1275 FrameState frameState = invoke.stateAfter(); 1276 invoke.asNode().safeDelete(); 1277 assert invoke.callTarget() == null : "must not have been added to the graph yet"; 1278 if (frameState != null && frameState.hasNoUsages()) { 1279 frameState.safeDelete(); 1280 } 1281 } 1282 1283 protected abstract EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, 1284 boolean trackNodeSourcePosition); 1285 1286 @SuppressWarnings("try") 1287 @Override 1288 protected Node canonicalizeFixedNode(MethodScope s, Node node) { 1289 PEMethodScope methodScope = (PEMethodScope) s; 1290 1291 Node replacedNode = node; 1292 if (nodePlugins != null && nodePlugins.length > 0) { 1293 if (node instanceof LoadFieldNode) { 1294 LoadFieldNode loadFieldNode = (LoadFieldNode) node; 1295 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, loadFieldNode); 1296 ResolvedJavaField field = loadFieldNode.field(); 1297 if (loadFieldNode.isStatic()) { 1298 for (NodePlugin nodePlugin : nodePlugins) { 1299 if (nodePlugin.handleLoadStaticField(graphBuilderContext, field)) { 1300 replacedNode = graphBuilderContext.pushedNode; 1301 break; 1302 } 1303 } 1304 } else { 1305 ValueNode object = loadFieldNode.object(); 1306 for (NodePlugin nodePlugin : nodePlugins) { 1307 if (nodePlugin.handleLoadField(graphBuilderContext, object, field)) { 1308 replacedNode = graphBuilderContext.pushedNode; 1309 break; 1310 } 1311 } 1312 } 1313 } else if (node instanceof StoreFieldNode) { 1314 StoreFieldNode storeFieldNode = (StoreFieldNode) node; 1315 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, storeFieldNode); 1316 ResolvedJavaField field = storeFieldNode.field(); 1317 if (storeFieldNode.isStatic()) { 1318 ValueNode value = storeFieldNode.value(); 1319 for (NodePlugin nodePlugin : nodePlugins) { 1320 if (nodePlugin.handleStoreStaticField(graphBuilderContext, field, value)) { 1321 replacedNode = graphBuilderContext.pushedNode; 1322 break; 1323 } 1324 } 1325 } else { 1326 ValueNode object = storeFieldNode.object(); 1327 ValueNode value = storeFieldNode.value(); 1328 for (NodePlugin nodePlugin : nodePlugins) { 1329 if (nodePlugin.handleStoreField(graphBuilderContext, object, field, value)) { 1330 replacedNode = graphBuilderContext.pushedNode; 1331 break; 1332 } 1333 } 1334 } 1335 } else if (node instanceof LoadIndexedNode) { 1336 LoadIndexedNode loadIndexedNode = (LoadIndexedNode) node; 1337 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, loadIndexedNode); 1338 ValueNode array = loadIndexedNode.array(); 1339 ValueNode index = loadIndexedNode.index(); 1340 for (NodePlugin nodePlugin : nodePlugins) { 1341 if (nodePlugin.handleLoadIndexed(graphBuilderContext, array, index, loadIndexedNode.getBoundsCheck(), loadIndexedNode.elementKind())) { 1342 replacedNode = graphBuilderContext.pushedNode; 1343 break; 1344 } 1345 } 1346 } else if (node instanceof StoreIndexedNode) { 1347 StoreIndexedNode storeIndexedNode = (StoreIndexedNode) node; 1348 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, storeIndexedNode); 1349 ValueNode array = storeIndexedNode.array(); 1350 ValueNode index = storeIndexedNode.index(); 1351 ValueNode value = storeIndexedNode.value(); 1352 for (NodePlugin nodePlugin : nodePlugins) { 1353 if (nodePlugin.handleStoreIndexed(graphBuilderContext, array, index, storeIndexedNode.getBoundsCheck(), storeIndexedNode.getStoreCheck(), storeIndexedNode.elementKind(), value)) { 1354 replacedNode = graphBuilderContext.pushedNode; 1355 break; 1356 } 1357 } 1358 } else if (node instanceof NewInstanceNode) { 1359 NewInstanceNode newInstanceNode = (NewInstanceNode) node; 1360 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, newInstanceNode); 1361 ResolvedJavaType type = newInstanceNode.instanceClass(); 1362 for (NodePlugin nodePlugin : nodePlugins) { 1363 if (nodePlugin.handleNewInstance(graphBuilderContext, type)) { 1364 replacedNode = graphBuilderContext.pushedNode; 1365 break; 1366 } 1367 } 1368 } else if (node instanceof NewArrayNode) { 1369 NewArrayNode newArrayNode = (NewArrayNode) node; 1370 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, newArrayNode); 1371 ResolvedJavaType elementType = newArrayNode.elementType(); 1372 ValueNode length = newArrayNode.length(); 1373 for (NodePlugin nodePlugin : nodePlugins) { 1374 if (nodePlugin.handleNewArray(graphBuilderContext, elementType, length)) { 1375 replacedNode = graphBuilderContext.pushedNode; 1376 break; 1377 } 1378 } 1379 } else if (node instanceof NewMultiArrayNode) { 1380 NewMultiArrayNode newArrayNode = (NewMultiArrayNode) node; 1381 PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(methodScope, newArrayNode); 1382 ResolvedJavaType elementType = newArrayNode.type(); 1383 ValueNode[] dimensions = newArrayNode.dimensions().toArray(new ValueNode[0]); 1384 for (NodePlugin nodePlugin : nodePlugins) { 1385 if (nodePlugin.handleNewMultiArray(graphBuilderContext, elementType, dimensions)) { 1386 replacedNode = graphBuilderContext.pushedNode; 1387 break; 1388 } 1389 } 1390 } 1391 } 1392 if (node instanceof PluginReplacementNode) { 1393 PluginReplacementNode pluginReplacementNode = (PluginReplacementNode) node; 1394 PEPluginGraphBuilderContext graphBuilderContext = new PEPluginGraphBuilderContext(methodScope, 1395 pluginReplacementNode); 1396 boolean success = pluginReplacementNode.replace(graphBuilderContext, providers.getReplacements()); 1397 if (success) { 1398 replacedNode = graphBuilderContext.pushedNode; 1399 } else if (pluginReplacementMustSucceed()) { 1400 throw new GraalError("Plugin failed:" + node); 1401 } 1402 } 1403 1404 return super.canonicalizeFixedNode(methodScope, replacedNode); 1405 } 1406 1407 protected boolean pluginReplacementMustSucceed() { 1408 return false; 1409 } 1410 1411 @Override 1412 protected Node handleFloatingNodeBeforeAdd(MethodScope s, LoopScope loopScope, Node n) { 1413 PEMethodScope methodScope = (PEMethodScope) s; 1414 1415 Node node = n; 1416 if (node instanceof ParameterNode) { 1417 ParameterNode param = (ParameterNode) node; 1418 if (methodScope.isInlinedMethod()) { 1419 throw GraalError.shouldNotReachHere("Parameter nodes are already registered when the inlined scope is created"); 1420 1421 } else if (parameterPlugin != null) { 1422 assert !methodScope.isInlinedMethod(); 1423 GraphBuilderContext graphBuilderContext = new PENonAppendGraphBuilderContext(methodScope, null); 1424 Node result = parameterPlugin.interceptParameter(graphBuilderContext, param.index(), 1425 StampPair.create(param.stamp(NodeView.DEFAULT), param.uncheckedStamp())); 1426 if (result != null) { 1427 return result; 1428 } 1429 } 1430 node = param.copyWithInputs(); 1431 } 1432 1433 return super.handleFloatingNodeBeforeAdd(methodScope, loopScope, node); 1434 } 1435 1436 protected void ensureOuterStateDecoded(PEMethodScope methodScope) { 1437 if (methodScope.outerState == null && methodScope.caller != null) { 1438 FrameState stateAtReturn = methodScope.invokeData.invoke.stateAfter(); 1439 if (stateAtReturn == null) { 1440 stateAtReturn = (FrameState) decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); 1441 } 1442 1443 JavaKind invokeReturnKind = methodScope.invokeData.invoke.asNode().getStackKind(); 1444 FrameState outerState = stateAtReturn.duplicateModified(graph, methodScope.invokeData.invoke.bci(), stateAtReturn.rethrowException(), true, invokeReturnKind, null, null); 1445 1446 /* 1447 * When the encoded graph has methods inlining, we can already have a proper caller 1448 * state. If not, we set the caller state here. 1449 */ 1450 if (outerState.outerFrameState() == null && methodScope.caller != null) { 1451 ensureOuterStateDecoded(methodScope.caller); 1452 outerState.setOuterFrameState(methodScope.caller.outerState); 1453 } 1454 methodScope.outerState = outerState; 1455 } 1456 } 1457 1458 protected void ensureStateAfterDecoded(PEMethodScope methodScope) { 1459 if (methodScope.invokeData.invoke.stateAfter() == null) { 1460 methodScope.invokeData.invoke.setStateAfter((FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId)); 1461 } 1462 } 1463 1464 protected void ensureExceptionStateDecoded(PEMethodScope methodScope) { 1465 if (methodScope.exceptionState == null && methodScope.caller != null && methodScope.invokeData.invoke instanceof InvokeWithExceptionNode) { 1466 ensureStateAfterDecoded(methodScope); 1467 1468 assert methodScope.exceptionPlaceholderNode == null; 1469 methodScope.exceptionPlaceholderNode = graph.add(new ExceptionPlaceholderNode()); 1470 registerNode(methodScope.callerLoopScope, methodScope.invokeData.exceptionOrderId, methodScope.exceptionPlaceholderNode, false, false); 1471 FrameState exceptionState = (FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.exceptionStateOrderId); 1472 1473 if (exceptionState.outerFrameState() == null && methodScope.caller != null) { 1474 ensureOuterStateDecoded(methodScope.caller); 1475 exceptionState.setOuterFrameState(methodScope.caller.outerState); 1476 } 1477 methodScope.exceptionState = exceptionState; 1478 } 1479 } 1480 1481 @Override 1482 protected Node handleFloatingNodeAfterAdd(MethodScope s, LoopScope loopScope, Node node) { 1483 PEMethodScope methodScope = (PEMethodScope) s; 1484 1485 if (methodScope.isInlinedMethod()) { 1486 if (node instanceof FrameState) { 1487 FrameState frameState = (FrameState) node; 1488 1489 ensureOuterStateDecoded(methodScope); 1490 if (frameState.bci < 0) { 1491 ensureExceptionStateDecoded(methodScope); 1492 } 1493 List<ValueNode> invokeArgsList = null; 1494 if (frameState.bci == BytecodeFrame.BEFORE_BCI) { 1495 /* 1496 * We know that the argument list is only used in this case, so avoid the List 1497 * allocation for "normal" bcis. 1498 */ 1499 invokeArgsList = Arrays.asList(methodScope.arguments); 1500 } 1501 return InliningUtil.processFrameState(frameState, methodScope.invokeData.invoke, null, methodScope.method, methodScope.exceptionState, methodScope.outerState, true, 1502 methodScope.method, invokeArgsList); 1503 1504 } else if (node instanceof MonitorIdNode) { 1505 ensureOuterStateDecoded(methodScope); 1506 InliningUtil.processMonitorId(methodScope.outerState, (MonitorIdNode) node); 1507 return node; 1508 } 1509 } 1510 1511 return node; 1512 } 1513 } 1514