1 /*
2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.
8  *
9  * This code is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * version 2 for more details (a copy is included in the LICENSE file that
13  * accompanied this code).
14  *
15  * You should have received a copy of the GNU General Public License version
16  * 2 along with this work; if not, write to the Free Software Foundation,
17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20  * or visit www.oracle.com if you need additional information or have any
21  * questions.
22  */
23 
24 
25 package org.graalvm.compiler.replacements;
26 
27 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_READ;
28 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_WRITE;
29 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_READ;
30 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_WRITE;
31 import static jdk.vm.ci.code.MemoryBarriers.LOAD_LOAD;
32 import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE;
33 import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
34 import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE;
35 import static org.graalvm.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION;
36 import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier;
37 
38 import java.lang.reflect.Array;
39 import java.lang.reflect.Field;
40 import java.util.Arrays;
41 
42 import org.graalvm.compiler.api.directives.GraalDirectives;
43 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
44 import org.graalvm.compiler.bytecode.BytecodeProvider;
45 import org.graalvm.compiler.core.common.calc.Condition;
46 import org.graalvm.compiler.core.common.calc.Condition.CanonicalizedCondition;
47 import org.graalvm.compiler.core.common.calc.UnsignedMath;
48 import org.graalvm.compiler.core.common.type.ObjectStamp;
49 import org.graalvm.compiler.core.common.type.Stamp;
50 import org.graalvm.compiler.core.common.type.StampFactory;
51 import org.graalvm.compiler.core.common.type.TypeReference;
52 import org.graalvm.compiler.debug.GraalError;
53 import org.graalvm.compiler.graph.Edges;
54 import org.graalvm.compiler.graph.Node;
55 import org.graalvm.compiler.graph.NodeList;
56 import org.graalvm.compiler.nodes.ConstantNode;
57 import org.graalvm.compiler.nodes.DeoptimizeNode;
58 import org.graalvm.compiler.nodes.EndNode;
59 import org.graalvm.compiler.nodes.FixedGuardNode;
60 import org.graalvm.compiler.nodes.FixedWithNextNode;
61 import org.graalvm.compiler.nodes.IfNode;
62 import org.graalvm.compiler.nodes.LogicNode;
63 import org.graalvm.compiler.nodes.MergeNode;
64 import org.graalvm.compiler.nodes.NodeView;
65 import org.graalvm.compiler.nodes.StateSplit;
66 import org.graalvm.compiler.nodes.StructuredGraph;
67 import org.graalvm.compiler.nodes.ValueNode;
68 import org.graalvm.compiler.nodes.ValuePhiNode;
69 import org.graalvm.compiler.nodes.calc.AbsNode;
70 import org.graalvm.compiler.nodes.calc.CompareNode;
71 import org.graalvm.compiler.nodes.calc.ConditionalNode;
72 import org.graalvm.compiler.nodes.calc.FloatEqualsNode;
73 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
74 import org.graalvm.compiler.nodes.calc.IsNullNode;
75 import org.graalvm.compiler.nodes.calc.NarrowNode;
76 import org.graalvm.compiler.nodes.calc.ReinterpretNode;
77 import org.graalvm.compiler.nodes.calc.RightShiftNode;
78 import org.graalvm.compiler.nodes.calc.SignExtendNode;
79 import org.graalvm.compiler.nodes.calc.SqrtNode;
80 import org.graalvm.compiler.nodes.calc.UnsignedDivNode;
81 import org.graalvm.compiler.nodes.calc.UnsignedRemNode;
82 import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
83 import org.graalvm.compiler.nodes.debug.BindToRegisterNode;
84 import org.graalvm.compiler.nodes.debug.BlackholeNode;
85 import org.graalvm.compiler.nodes.debug.ControlFlowAnchorNode;
86 import org.graalvm.compiler.nodes.debug.OpaqueNode;
87 import org.graalvm.compiler.nodes.debug.SpillRegistersNode;
88 import org.graalvm.compiler.nodes.extended.BoxNode;
89 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
90 import org.graalvm.compiler.nodes.extended.GetClassNode;
91 import org.graalvm.compiler.nodes.extended.MembarNode;
92 import org.graalvm.compiler.nodes.extended.RawLoadNode;
93 import org.graalvm.compiler.nodes.extended.RawStoreNode;
94 import org.graalvm.compiler.nodes.extended.UnboxNode;
95 import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode;
96 import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode;
97 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
98 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
99 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver;
100 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
101 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
102 import org.graalvm.compiler.nodes.java.ClassIsAssignableFromNode;
103 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
104 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
105 import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode;
106 import org.graalvm.compiler.nodes.java.LoadFieldNode;
107 import org.graalvm.compiler.nodes.java.RegisterFinalizerNode;
108 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode;
109 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode;
110 import org.graalvm.compiler.nodes.type.StampTool;
111 import org.graalvm.compiler.nodes.util.GraphUtil;
112 import org.graalvm.compiler.nodes.virtual.EnsureVirtualizedNode;
113 import org.graalvm.compiler.replacements.nodes.ReverseBytesNode;
114 import org.graalvm.compiler.replacements.nodes.VirtualizableInvokeMacroNode;
115 import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactNode;
116 import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactNode;
117 import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactNode;
118 import jdk.internal.vm.compiler.word.LocationIdentity;
119 
120 import jdk.vm.ci.code.BytecodePosition;
121 import jdk.vm.ci.meta.DeoptimizationAction;
122 import jdk.vm.ci.meta.DeoptimizationReason;
123 import jdk.vm.ci.meta.JavaConstant;
124 import jdk.vm.ci.meta.JavaKind;
125 import jdk.vm.ci.meta.MetaAccessProvider;
126 import jdk.vm.ci.meta.ResolvedJavaField;
127 import jdk.vm.ci.meta.ResolvedJavaMethod;
128 import jdk.vm.ci.meta.ResolvedJavaType;
129 import jdk.vm.ci.meta.SpeculationLog;
130 import jdk.vm.ci.meta.SpeculationLog.Speculation;
131 import sun.misc.Unsafe;
132 
133 /**
134  * Provides non-runtime specific {@link InvocationPlugin}s.
135  */
136 public class StandardGraphBuilderPlugins {
137 
registerInvocationPlugins(MetaAccessProvider metaAccess, SnippetReflectionProvider snippetReflection, InvocationPlugins plugins, BytecodeProvider bytecodeProvider, boolean allowDeoptimization, boolean explicitUnsafeNullChecks)138     public static void registerInvocationPlugins(MetaAccessProvider metaAccess, SnippetReflectionProvider snippetReflection, InvocationPlugins plugins, BytecodeProvider bytecodeProvider,
139                     boolean allowDeoptimization, boolean explicitUnsafeNullChecks) {
140         registerObjectPlugins(plugins);
141         registerClassPlugins(plugins);
142         registerMathPlugins(plugins, allowDeoptimization);
143         registerUnsignedMathPlugins(plugins);
144         registerStringPlugins(plugins, bytecodeProvider, snippetReflection);
145         registerCharacterPlugins(plugins);
146         registerShortPlugins(plugins);
147         registerIntegerLongPlugins(plugins, JavaKind.Int);
148         registerIntegerLongPlugins(plugins, JavaKind.Long);
149         registerFloatPlugins(plugins);
150         registerDoublePlugins(plugins);
151         registerArraysPlugins(plugins, bytecodeProvider);
152         registerArrayPlugins(plugins, bytecodeProvider);
153         registerUnsafePlugins(plugins, bytecodeProvider, explicitUnsafeNullChecks);
154         registerEdgesPlugins(metaAccess, plugins);
155         registerGraalDirectivesPlugins(plugins);
156         registerBoxingPlugins(plugins);
157         registerJMHBlackholePlugins(plugins, bytecodeProvider);
158         registerJFRThrowablePlugins(plugins, bytecodeProvider);
159         registerMethodHandleImplPlugins(plugins, snippetReflection, bytecodeProvider);
160         registerJcovCollectPlugins(plugins, bytecodeProvider);
161     }
162 
163     private static final Field STRING_VALUE_FIELD;
164 
165     static {
166         try {
167             STRING_VALUE_FIELD = String.class.getDeclaredField("value");
168         } catch (NoSuchFieldException e) {
169             throw new GraalError(e);
170         }
171     }
172 
registerStringPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, SnippetReflectionProvider snippetReflection)173     private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, SnippetReflectionProvider snippetReflection) {
174         final Registration r = new Registration(plugins, String.class, bytecodeProvider);
175         r.register1("hashCode", Receiver.class, new InvocationPlugin() {
176             @Override
177             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
178                 if (receiver.isConstant()) {
179                     String s = snippetReflection.asObject(String.class, (JavaConstant) receiver.get().asConstant());
180                     b.addPush(JavaKind.Int, b.add(ConstantNode.forInt(s.hashCode())));
181                     return true;
182                 }
183                 return false;
184             }
185         });
186 
187         if (Java8OrEarlier) {
188             r.registerMethodSubstitution(StringSubstitutions.class, "equals", Receiver.class, Object.class);
189             r.register7("indexOf", char[].class, int.class, int.class, char[].class, int.class, int.class, int.class, new StringIndexOfConstantPlugin());
190 
191             Registration sr = new Registration(plugins, StringSubstitutions.class);
192             sr.register1("getValue", String.class, new InvocationPlugin() {
193                 @Override
194                 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
195                     ResolvedJavaField field = b.getMetaAccess().lookupJavaField(STRING_VALUE_FIELD);
196                     b.addPush(JavaKind.Object, LoadFieldNode.create(b.getConstantFieldProvider(), b.getConstantReflection(), b.getMetaAccess(),
197                                     b.getOptions(), b.getAssumptions(), value, field, false, false));
198                     return true;
199                 }
200             });
201         }
202     }
203 
registerArraysPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider)204     private static void registerArraysPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
205         Registration r = new Registration(plugins, Arrays.class, bytecodeProvider);
206         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", boolean[].class, boolean[].class);
207         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", byte[].class, byte[].class);
208         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", short[].class, short[].class);
209         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", char[].class, char[].class);
210         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", int[].class, int[].class);
211         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", long[].class, long[].class);
212     }
213 
registerArrayPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider)214     private static void registerArrayPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
215         Registration r = new Registration(plugins, Array.class, bytecodeProvider);
216         r.register2("newInstance", Class.class, int.class, new InvocationPlugin() {
217             @Override
218             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unused, ValueNode componentType, ValueNode length) {
219                 b.addPush(JavaKind.Object, new DynamicNewArrayNode(componentType, length, true));
220                 return true;
221             }
222         });
223         r.registerMethodSubstitution(ArraySubstitutions.class, "getLength", Object.class);
224     }
225 
226     private abstract static class UnsafeCompareAndUpdatePluginsRegistrar {
register(Registration r, String casPrefix, boolean explicitUnsafeNullChecks, JavaKind[] compareAndSwapTypes)227         public void register(Registration r, String casPrefix, boolean explicitUnsafeNullChecks, JavaKind[] compareAndSwapTypes) {
228             for (JavaKind kind : compareAndSwapTypes) {
229                 Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
230                 r.register5(casPrefix + kind.name(), Receiver.class, Object.class, long.class, javaClass, javaClass, new UnsafeAccessPlugin(returnKind(kind), explicitUnsafeNullChecks) {
231                     @Override
232                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode expected, ValueNode x) {
233                         // Emits a null-check for the otherwise unused receiver
234                         unsafe.get();
235                         createUnsafeAccess(object, b, (obj, loc) -> UnsafeCompareAndUpdatePluginsRegistrar.this.createNode(obj, offset, expected, x, kind, loc));
236                         return true;
237                     }
238                 });
239             }
240         }
241 
createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity)242         public abstract FixedWithNextNode createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity);
243 
returnKind(JavaKind accessKind)244         public abstract JavaKind returnKind(JavaKind accessKind);
245     }
246 
247     private static class UnsafeCompareAndSwapPluginsRegistrar extends UnsafeCompareAndUpdatePluginsRegistrar {
248         @Override
createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity)249         public FixedWithNextNode createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity) {
250             return new UnsafeCompareAndSwapNode(object, offset, expected, newValue, kind, identity);
251         }
252 
253         @Override
returnKind(JavaKind accessKind)254         public JavaKind returnKind(JavaKind accessKind) {
255             return JavaKind.Boolean.getStackKind();
256         }
257     }
258 
259     private static UnsafeCompareAndSwapPluginsRegistrar unsafeCompareAndSwapPluginsRegistrar = new UnsafeCompareAndSwapPluginsRegistrar();
260 
261     private static class UnsafeCompareAndExchangePluginsRegistrar extends UnsafeCompareAndUpdatePluginsRegistrar {
262         @Override
createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity)263         public FixedWithNextNode createNode(ValueNode object, ValueNode offset, ValueNode expected, ValueNode newValue, JavaKind kind, LocationIdentity identity) {
264             return new UnsafeCompareAndExchangeNode(object, offset, expected, newValue, kind, identity);
265         }
266 
267         @Override
returnKind(JavaKind accessKind)268         public JavaKind returnKind(JavaKind accessKind) {
269             if (accessKind.isNumericInteger()) {
270                 return accessKind.getStackKind();
271             } else {
272                 return accessKind;
273             }
274         }
275     }
276 
277     private static UnsafeCompareAndExchangePluginsRegistrar unsafeCompareAndExchangePluginsRegistrar = new UnsafeCompareAndExchangePluginsRegistrar();
278 
registerPlatformSpecificUnsafePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, boolean explicitUnsafeNullChecks, JavaKind[] supportedCasKinds)279     public static void registerPlatformSpecificUnsafePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, boolean explicitUnsafeNullChecks, JavaKind[] supportedCasKinds) {
280         registerPlatformSpecificUnsafePlugins(supportedCasKinds, new Registration(plugins, Unsafe.class), true, explicitUnsafeNullChecks);
281         if (!Java8OrEarlier) {
282             registerPlatformSpecificUnsafePlugins(supportedCasKinds, new Registration(plugins, "jdk.internal.misc.Unsafe", bytecodeProvider), false, explicitUnsafeNullChecks);
283         }
284 
285     }
286 
registerPlatformSpecificUnsafePlugins(JavaKind[] supportedCasKinds, Registration r, boolean java8OrEarlier, boolean explicitUnsafeNullChecks)287     private static void registerPlatformSpecificUnsafePlugins(JavaKind[] supportedCasKinds, Registration r, boolean java8OrEarlier, boolean explicitUnsafeNullChecks) {
288         if (java8OrEarlier) {
289             unsafeCompareAndSwapPluginsRegistrar.register(r, "compareAndSwap", explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object});
290         } else {
291             unsafeCompareAndSwapPluginsRegistrar.register(r, "compareAndSet", explicitUnsafeNullChecks, supportedCasKinds);
292             unsafeCompareAndExchangePluginsRegistrar.register(r, "compareAndExchange", explicitUnsafeNullChecks, supportedCasKinds);
293         }
294     }
295 
registerUnsafePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, boolean explicitUnsafeNullChecks)296     private static void registerUnsafePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider, boolean explicitUnsafeNullChecks) {
297         registerUnsafePlugins(new Registration(plugins, Unsafe.class), true, explicitUnsafeNullChecks);
298         if (!Java8OrEarlier) {
299             registerUnsafePlugins(new Registration(plugins, "jdk.internal.misc.Unsafe", bytecodeProvider), false, explicitUnsafeNullChecks);
300         }
301     }
302 
registerUnsafePlugins(Registration r, boolean java8OrEarlier, boolean explicitUnsafeNullChecks)303     private static void registerUnsafePlugins(Registration r, boolean java8OrEarlier, boolean explicitUnsafeNullChecks) {
304         for (JavaKind kind : JavaKind.values()) {
305             if ((kind.isPrimitive() && kind != JavaKind.Void) || kind == JavaKind.Object) {
306                 Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
307                 String kindName = kind.name();
308                 String getName = "get" + kindName;
309                 String putName = "put" + kindName;
310                 // Object-based accesses
311                 r.register3(getName, Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, explicitUnsafeNullChecks));
312                 r.register4(putName, Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, explicitUnsafeNullChecks));
313                 // Volatile object-based accesses
314                 r.register3(getName + "Volatile", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, AccessKind.VOLATILE, explicitUnsafeNullChecks));
315                 r.register4(putName + "Volatile", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, AccessKind.VOLATILE, explicitUnsafeNullChecks));
316                 // Ordered object-based accesses
317                 if (java8OrEarlier) {
318                     if (kind == JavaKind.Int || kind == JavaKind.Long || kind == JavaKind.Object) {
319                         r.register4("putOrdered" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, AccessKind.RELEASE_ACQUIRE, explicitUnsafeNullChecks));
320                     }
321                 } else {
322                     r.register4("put" + kindName + "Release", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, AccessKind.RELEASE_ACQUIRE, explicitUnsafeNullChecks));
323                     r.register3("get" + kindName + "Acquire", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, AccessKind.RELEASE_ACQUIRE, explicitUnsafeNullChecks));
324                     r.register4("put" + kindName + "Opaque", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, AccessKind.OPAQUE, explicitUnsafeNullChecks));
325                     r.register3("get" + kindName + "Opaque", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, AccessKind.OPAQUE, explicitUnsafeNullChecks));
326                 }
327                 if (kind != JavaKind.Boolean && kind != JavaKind.Object) {
328                     // Raw accesses to memory addresses
329                     r.register2(getName, Receiver.class, long.class, new UnsafeGetPlugin(kind, explicitUnsafeNullChecks));
330                     r.register3(putName, Receiver.class, long.class, kind.toJavaClass(), new UnsafePutPlugin(kind, explicitUnsafeNullChecks));
331                 }
332             }
333         }
334 
335         // Accesses to native memory addresses.
336         r.register2("getAddress", Receiver.class, long.class, new UnsafeGetPlugin(JavaKind.Long, explicitUnsafeNullChecks));
337         r.register3("putAddress", Receiver.class, long.class, long.class, new UnsafePutPlugin(JavaKind.Long, explicitUnsafeNullChecks));
338 
339         r.register2("allocateInstance", Receiver.class, Class.class, new InvocationPlugin() {
340 
341             @Override
342             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode clazz) {
343                 // Emits a null-check for the otherwise unused receiver
344                 unsafe.get();
345                 b.addPush(JavaKind.Object, new DynamicNewInstanceNode(b.nullCheckedValue(clazz, DeoptimizationAction.None), true));
346                 return true;
347             }
348 
349         });
350 
351         r.register1("loadFence", Receiver.class, new UnsafeFencePlugin(LOAD_LOAD | LOAD_STORE));
352         r.register1("storeFence", Receiver.class, new UnsafeFencePlugin(STORE_STORE | LOAD_STORE));
353         r.register1("fullFence", Receiver.class, new UnsafeFencePlugin(LOAD_LOAD | STORE_STORE | LOAD_STORE | STORE_LOAD));
354     }
355 
registerIntegerLongPlugins(InvocationPlugins plugins, JavaKind kind)356     private static void registerIntegerLongPlugins(InvocationPlugins plugins, JavaKind kind) {
357         Class<?> declaringClass = kind.toBoxedJavaClass();
358         Class<?> type = kind.toJavaClass();
359         Registration r = new Registration(plugins, declaringClass);
360         r.register1("reverseBytes", type, new InvocationPlugin() {
361             @Override
362             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
363                 b.push(kind, b.append(new ReverseBytesNode(value).canonical(null)));
364                 return true;
365             }
366         });
367         r.register2("divideUnsigned", type, type, new InvocationPlugin() {
368             @Override
369             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode dividend, ValueNode divisor) {
370                 b.push(kind, b.append(UnsignedDivNode.create(dividend, divisor, null, NodeView.DEFAULT)));
371                 return true;
372             }
373         });
374         r.register2("remainderUnsigned", type, type, new InvocationPlugin() {
375             @Override
376             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode dividend, ValueNode divisor) {
377                 b.push(kind, b.append(UnsignedRemNode.create(dividend, divisor, null, NodeView.DEFAULT)));
378                 return true;
379             }
380         });
381     }
382 
registerCharacterPlugins(InvocationPlugins plugins)383     private static void registerCharacterPlugins(InvocationPlugins plugins) {
384         Registration r = new Registration(plugins, Character.class);
385         r.register1("reverseBytes", char.class, new InvocationPlugin() {
386             @Override
387             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
388                 // return (char) (Integer.reverse(i) >> 16);
389                 ReverseBytesNode reverse = b.add(new ReverseBytesNode(value));
390                 RightShiftNode rightShift = b.add(new RightShiftNode(reverse, b.add(ConstantNode.forInt(16))));
391                 ZeroExtendNode charCast = b.add(new ZeroExtendNode(b.add(new NarrowNode(rightShift, 16)), 32));
392                 b.push(JavaKind.Char, b.append(charCast.canonical(null)));
393                 return true;
394             }
395         });
396     }
397 
registerShortPlugins(InvocationPlugins plugins)398     private static void registerShortPlugins(InvocationPlugins plugins) {
399         Registration r = new Registration(plugins, Short.class);
400         r.register1("reverseBytes", short.class, new InvocationPlugin() {
401             @Override
402             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
403                 // return (short) (Integer.reverse(i) >> 16);
404                 ReverseBytesNode reverse = b.add(new ReverseBytesNode(value));
405                 RightShiftNode rightShift = b.add(new RightShiftNode(reverse, b.add(ConstantNode.forInt(16))));
406                 SignExtendNode charCast = b.add(new SignExtendNode(b.add(new NarrowNode(rightShift, 16)), 32));
407                 b.push(JavaKind.Short, b.append(charCast.canonical(null)));
408                 return true;
409             }
410         });
411     }
412 
registerFloatPlugins(InvocationPlugins plugins)413     private static void registerFloatPlugins(InvocationPlugins plugins) {
414         Registration r = new Registration(plugins, Float.class);
415         r.register1("floatToRawIntBits", float.class, new InvocationPlugin() {
416             @Override
417             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
418                 b.push(JavaKind.Int, b.append(ReinterpretNode.create(JavaKind.Int, value, NodeView.DEFAULT)));
419                 return true;
420             }
421         });
422         r.register1("floatToIntBits", float.class, new InvocationPlugin() {
423             @Override
424             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
425                 LogicNode notNan = b.append(FloatEqualsNode.create(value, value, NodeView.DEFAULT));
426                 ValueNode raw = b.append(ReinterpretNode.create(JavaKind.Int, value, NodeView.DEFAULT));
427                 ValueNode result = b.append(ConditionalNode.create(notNan, raw, ConstantNode.forInt(0x7fc00000), NodeView.DEFAULT));
428                 b.push(JavaKind.Int, result);
429                 return true;
430             }
431         });
432         r.register1("intBitsToFloat", int.class, new InvocationPlugin() {
433             @Override
434             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
435                 b.push(JavaKind.Float, b.append(ReinterpretNode.create(JavaKind.Float, value, NodeView.DEFAULT)));
436                 return true;
437             }
438         });
439     }
440 
registerDoublePlugins(InvocationPlugins plugins)441     private static void registerDoublePlugins(InvocationPlugins plugins) {
442         Registration r = new Registration(plugins, Double.class);
443         r.register1("doubleToRawLongBits", double.class, new InvocationPlugin() {
444             @Override
445             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
446                 b.push(JavaKind.Long, b.append(ReinterpretNode.create(JavaKind.Long, value, NodeView.DEFAULT)));
447                 return true;
448             }
449         });
450         r.register1("doubleToLongBits", double.class, new InvocationPlugin() {
451             @Override
452             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
453                 LogicNode notNan = b.append(FloatEqualsNode.create(value, value, NodeView.DEFAULT));
454                 ValueNode raw = b.append(ReinterpretNode.create(JavaKind.Long, value, NodeView.DEFAULT));
455                 ValueNode result = b.append(ConditionalNode.create(notNan, raw, ConstantNode.forLong(0x7ff8000000000000L), NodeView.DEFAULT));
456                 b.push(JavaKind.Long, result);
457                 return true;
458             }
459         });
460         r.register1("longBitsToDouble", long.class, new InvocationPlugin() {
461             @Override
462             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
463                 b.push(JavaKind.Double, b.append(ReinterpretNode.create(JavaKind.Double, value, NodeView.DEFAULT)));
464                 return true;
465             }
466         });
467     }
468 
registerMathPlugins(InvocationPlugins plugins, boolean allowDeoptimization)469     private static void registerMathPlugins(InvocationPlugins plugins, boolean allowDeoptimization) {
470         Registration r = new Registration(plugins, Math.class);
471         if (allowDeoptimization) {
472             for (JavaKind kind : new JavaKind[]{JavaKind.Int, JavaKind.Long}) {
473                 Class<?> type = kind.toJavaClass();
474 
475                 r.register1("decrementExact", type, new InvocationPlugin() {
476                     @Override
477                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) {
478                         b.addPush(kind, new IntegerSubExactNode(x, ConstantNode.forIntegerKind(kind, 1)));
479                         return true;
480                     }
481                 });
482 
483                 r.register1("incrementExact", type, new InvocationPlugin() {
484                     @Override
485                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) {
486                         b.addPush(kind, new IntegerAddExactNode(x, ConstantNode.forIntegerKind(kind, 1)));
487                         return true;
488                     }
489                 });
490 
491                 r.register2("addExact", type, type, new InvocationPlugin() {
492                     @Override
493                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
494                         b.addPush(kind, new IntegerAddExactNode(x, y));
495                         return true;
496                     }
497                 });
498 
499                 r.register2("subtractExact", type, type, new InvocationPlugin() {
500                     @Override
501                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
502                         b.addPush(kind, new IntegerSubExactNode(x, y));
503                         return true;
504                     }
505                 });
506 
507                 r.register2("multiplyExact", type, type, new InvocationPlugin() {
508                     @Override
509                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
510                         b.addPush(kind, new IntegerMulExactNode(x, y));
511                         return true;
512                     }
513                 });
514             }
515         }
516         r.register1("abs", Float.TYPE, new InvocationPlugin() {
517 
518             @Override
519             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
520                 b.push(JavaKind.Float, b.append(new AbsNode(value).canonical(null)));
521                 return true;
522             }
523         });
524         r.register1("abs", Double.TYPE, new InvocationPlugin() {
525             @Override
526             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
527                 b.push(JavaKind.Double, b.append(new AbsNode(value).canonical(null)));
528                 return true;
529             }
530         });
531         r.register1("sqrt", Double.TYPE, new InvocationPlugin() {
532             @Override
533             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
534                 b.push(JavaKind.Double, b.append(SqrtNode.create(value, NodeView.DEFAULT)));
535                 return true;
536             }
537         });
538     }
539 
540     public static final class StringIndexOfConstantPlugin implements InvocationPlugin {
541         @Override
inlineOnly()542         public boolean inlineOnly() {
543             return true;
544         }
545 
546         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, InvocationPlugin.Receiver receiver, ValueNode source, ValueNode sourceOffset, ValueNode sourceCount, ValueNode target, ValueNode targetOffset, ValueNode targetCount, ValueNode origFromIndex)547         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, InvocationPlugin.Receiver receiver, ValueNode source, ValueNode sourceOffset, ValueNode sourceCount,
548                         ValueNode target, ValueNode targetOffset, ValueNode targetCount, ValueNode origFromIndex) {
549             if (target.isConstant()) {
550                 b.addPush(JavaKind.Int, new StringIndexOfNode(b.getInvokeKind(), targetMethod, b.bci(), b.getInvokeReturnStamp(b.getAssumptions()), source, sourceOffset, sourceCount,
551                                 target, targetOffset, targetCount, origFromIndex));
552                 return true;
553             }
554             return false;
555         }
556     }
557 
558     public static class UnsignedMathPlugin implements InvocationPlugin {
559         private final Condition condition;
560 
UnsignedMathPlugin(Condition condition)561         public UnsignedMathPlugin(Condition condition) {
562             this.condition = condition;
563         }
564 
565         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y)566         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
567             CanonicalizedCondition canonical = condition.canonicalize();
568             StructuredGraph graph = b.getGraph();
569 
570             ValueNode lhs = canonical.mustMirror() ? y : x;
571             ValueNode rhs = canonical.mustMirror() ? x : y;
572 
573             ValueNode trueValue = ConstantNode.forBoolean(!canonical.mustNegate(), graph);
574             ValueNode falseValue = ConstantNode.forBoolean(canonical.mustNegate(), graph);
575 
576             LogicNode compare = CompareNode.createCompareNode(graph, b.getConstantReflection(), b.getMetaAccess(), b.getOptions(), null, canonical.getCanonicalCondition(), lhs, rhs, NodeView.DEFAULT);
577             b.addPush(JavaKind.Boolean, new ConditionalNode(compare, trueValue, falseValue));
578             return true;
579         }
580     }
581 
registerUnsignedMathPlugins(InvocationPlugins plugins)582     private static void registerUnsignedMathPlugins(InvocationPlugins plugins) {
583         Registration r = new Registration(plugins, UnsignedMath.class);
584         r.register2("aboveThan", int.class, int.class, new UnsignedMathPlugin(Condition.AT));
585         r.register2("aboveThan", long.class, long.class, new UnsignedMathPlugin(Condition.AT));
586         r.register2("belowThan", int.class, int.class, new UnsignedMathPlugin(Condition.BT));
587         r.register2("belowThan", long.class, long.class, new UnsignedMathPlugin(Condition.BT));
588         r.register2("aboveOrEqual", int.class, int.class, new UnsignedMathPlugin(Condition.AE));
589         r.register2("aboveOrEqual", long.class, long.class, new UnsignedMathPlugin(Condition.AE));
590         r.register2("belowOrEqual", int.class, int.class, new UnsignedMathPlugin(Condition.BE));
591         r.register2("belowOrEqual", long.class, long.class, new UnsignedMathPlugin(Condition.BE));
592     }
593 
registerBoxingPlugins(InvocationPlugins plugins)594     protected static void registerBoxingPlugins(InvocationPlugins plugins) {
595         for (JavaKind kind : JavaKind.values()) {
596             if (kind.isPrimitive() && kind != JavaKind.Void) {
597                 new BoxPlugin(kind).register(plugins);
598                 new UnboxPlugin(kind).register(plugins);
599             }
600         }
601     }
602 
registerObjectPlugins(InvocationPlugins plugins)603     private static void registerObjectPlugins(InvocationPlugins plugins) {
604         Registration r = new Registration(plugins, Object.class);
605         r.register1("<init>", Receiver.class, new InvocationPlugin() {
606             @Override
607             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
608                 /*
609                  * Object.<init> is a common instrumentation point so only perform this rewrite if
610                  * the current definition is the normal empty method with a single return bytecode.
611                  * The finalizer registration will instead be performed by the BytecodeParser.
612                  */
613                 if (targetMethod.getCodeSize() == 1) {
614                     ValueNode object = receiver.get();
615                     if (RegisterFinalizerNode.mayHaveFinalizer(object, b.getAssumptions())) {
616                         b.add(new RegisterFinalizerNode(object));
617                     }
618                     return true;
619                 }
620                 return false;
621             }
622         });
623         r.register1("getClass", Receiver.class, new InvocationPlugin() {
624             @Override
625             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
626                 ValueNode object = receiver.get();
627                 ValueNode folded = GetClassNode.tryFold(b.getMetaAccess(), b.getConstantReflection(), NodeView.DEFAULT, GraphUtil.originalValue(object));
628                 if (folded != null) {
629                     b.addPush(JavaKind.Object, folded);
630                 } else {
631                     Stamp stamp = StampFactory.objectNonNull(TypeReference.createTrusted(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Class.class)));
632                     b.addPush(JavaKind.Object, new GetClassNode(stamp, object));
633                 }
634                 return true;
635             }
636         });
637     }
638 
registerClassPlugins(InvocationPlugins plugins)639     private static void registerClassPlugins(InvocationPlugins plugins) {
640         Registration r = new Registration(plugins, Class.class);
641         r.register2("isInstance", Receiver.class, Object.class, new InvocationPlugin() {
642             @Override
643             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver type, ValueNode object) {
644                 LogicNode condition = b.append(InstanceOfDynamicNode.create(b.getAssumptions(), b.getConstantReflection(), type.get(), object, false));
645                 b.push(JavaKind.Boolean, b.append(new ConditionalNode(condition).canonical(null)));
646                 return true;
647             }
648         });
649         r.register2("isAssignableFrom", Receiver.class, Class.class, new InvocationPlugin() {
650             @Override
651             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver type, ValueNode otherType) {
652                 ClassIsAssignableFromNode condition = b.append(new ClassIsAssignableFromNode(type.get(), otherType));
653                 b.push(JavaKind.Boolean, b.append(new ConditionalNode(condition).canonical(null)));
654                 return true;
655             }
656         });
657     }
658 
659     /**
660      * Substitutions for improving the performance of some critical methods in {@link Edges}. These
661      * substitutions improve the performance by forcing the relevant methods to be inlined
662      * (intrinsification being a special form of inlining) and removing a checked cast.
663      */
registerEdgesPlugins(MetaAccessProvider metaAccess, InvocationPlugins plugins)664     private static void registerEdgesPlugins(MetaAccessProvider metaAccess, InvocationPlugins plugins) {
665         Registration r = new Registration(plugins, Edges.class);
666         for (Class<?> c : new Class<?>[]{Node.class, NodeList.class}) {
667             r.register2("get" + c.getSimpleName() + "Unsafe", Node.class, long.class, new InvocationPlugin() {
668                 @Override
669                 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode node, ValueNode offset) {
670                     ObjectStamp stamp = StampFactory.object(TypeReference.createTrusted(b.getAssumptions(), metaAccess.lookupJavaType(c)));
671                     RawLoadNode value = b.add(new RawLoadNode(stamp, node, offset, LocationIdentity.any(), JavaKind.Object));
672                     b.addPush(JavaKind.Object, value);
673                     return true;
674                 }
675             });
676             r.register3("put" + c.getSimpleName() + "Unsafe", Node.class, long.class, c, new InvocationPlugin() {
677                 @Override
678                 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode node, ValueNode offset, ValueNode value) {
679                     b.add(new RawStoreNode(node, offset, value, JavaKind.Object, LocationIdentity.any()));
680                     return true;
681                 }
682             });
683         }
684     }
685 
686     public static class BoxPlugin implements InvocationPlugin {
687 
688         private final JavaKind kind;
689 
BoxPlugin(JavaKind kind)690         BoxPlugin(JavaKind kind) {
691             this.kind = kind;
692         }
693 
694         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value)695         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
696             if (b.parsingIntrinsic()) {
697                 ResolvedJavaMethod rootMethod = b.getGraph().method();
698                 if (b.getMetaAccess().lookupJavaType(BoxingSnippets.class).isAssignableFrom(rootMethod.getDeclaringClass())) {
699                     // Disable invocation plugins for boxing snippets so that the
700                     // original JDK methods are inlined
701                     return false;
702                 }
703             }
704             ResolvedJavaType resultType = b.getMetaAccess().lookupJavaType(kind.toBoxedJavaClass());
705             b.addPush(JavaKind.Object, new BoxNode(value, resultType, kind));
706             return true;
707         }
708 
register(InvocationPlugins plugins)709         void register(InvocationPlugins plugins) {
710             plugins.register(this, kind.toBoxedJavaClass(), "valueOf", kind.toJavaClass());
711         }
712     }
713 
714     public static class UnboxPlugin implements InvocationPlugin {
715 
716         private final JavaKind kind;
717 
UnboxPlugin(JavaKind kind)718         UnboxPlugin(JavaKind kind) {
719             this.kind = kind;
720         }
721 
722         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver)723         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
724             if (b.parsingIntrinsic()) {
725                 ResolvedJavaMethod rootMethod = b.getGraph().method();
726                 if (b.getMetaAccess().lookupJavaType(BoxingSnippets.class).isAssignableFrom(rootMethod.getDeclaringClass())) {
727                     // Disable invocation plugins for unboxing snippets so that the
728                     // original JDK methods are inlined
729                     return false;
730                 }
731             }
732             ValueNode valueNode = UnboxNode.create(b.getMetaAccess(), b.getConstantReflection(), receiver.get(), kind);
733             b.addPush(kind, valueNode);
734             return true;
735         }
736 
register(InvocationPlugins plugins)737         void register(InvocationPlugins plugins) {
738             String name = kind.toJavaClass().getSimpleName() + "Value";
739             plugins.register(this, kind.toBoxedJavaClass(), name, Receiver.class);
740         }
741     }
742 
743     /**
744      * The new memory order modes (JDK9+) are defined with cumulative effect, from weakest to
745      * strongest: Plain, Opaque, Release/Acquire, and Volatile. The existing Plain and Volatile
746      * modes are defined compatibly with their pre-JDK 9 forms. Any guaranteed property of a weaker
747      * mode, plus more, holds for a stronger mode. (Conversely, implementations are allowed to use a
748      * stronger mode than requested for any access.) In JDK 9, these are provided without a full
749      * formal specification.
750      */
751     enum AccessKind {
752         PLAIN(0, 0, 0, 0, false),
753         /**
754          * Opaque accesses are wrapped by dummy membars to avoid floating/hoisting, this is stronger
755          * than required since Opaque mode does not directly impose any ordering constraints with
756          * respect to other variables beyond Plain mode.
757          */
758         OPAQUE(0, 0, 0, 0, true),
759         RELEASE_ACQUIRE(0, LOAD_LOAD | LOAD_STORE, LOAD_STORE | STORE_STORE, 0, true),
760         VOLATILE(JMM_PRE_VOLATILE_READ, JMM_POST_VOLATILE_READ, JMM_PRE_VOLATILE_WRITE, JMM_POST_VOLATILE_WRITE, true);
761 
762         public final boolean emitBarriers;
763         public final int preReadBarriers;
764         public final int postReadBarriers;
765         public final int preWriteBarriers;
766         public final int postWriteBarriers;
767 
AccessKind(int preReadBarriers, int postReadBarriers, int preWriteBarriers, int postWriteBarriers, boolean emitBarriers)768         AccessKind(int preReadBarriers, int postReadBarriers, int preWriteBarriers, int postWriteBarriers, boolean emitBarriers) {
769             this.emitBarriers = emitBarriers;
770             this.preReadBarriers = preReadBarriers;
771             this.postReadBarriers = postReadBarriers;
772             this.preWriteBarriers = preWriteBarriers;
773             this.postWriteBarriers = postWriteBarriers;
774         }
775     }
776 
777     /**
778      * Unsafe access relative to null object is an access to off-heap memory. As linear pointer
779      * compression uses non-zero null, here null object must be replaced with zero constant.
780      */
781     public abstract static class UnsafeAccessPlugin implements InvocationPlugin {
782         @FunctionalInterface
783         public interface UnsafeNodeConstructor {
create(ValueNode value, LocationIdentity location)784             FixedWithNextNode create(ValueNode value, LocationIdentity location);
785         }
786 
787         protected final JavaKind unsafeAccessKind;
788         private final boolean explicitUnsafeNullChecks;
789 
UnsafeAccessPlugin(JavaKind kind, boolean explicitUnsafeNullChecks)790         public UnsafeAccessPlugin(JavaKind kind, boolean explicitUnsafeNullChecks) {
791             unsafeAccessKind = kind;
792             this.explicitUnsafeNullChecks = explicitUnsafeNullChecks;
793         }
794 
createObjectAccessNode(ValueNode value, UnsafeNodeConstructor nodeConstructor)795         private static FixedWithNextNode createObjectAccessNode(ValueNode value, UnsafeNodeConstructor nodeConstructor) {
796             return nodeConstructor.create(value, LocationIdentity.ANY_LOCATION);
797         }
798 
createMemoryAccessNode(StructuredGraph graph, UnsafeNodeConstructor nodeConstructor)799         private static FixedWithNextNode createMemoryAccessNode(StructuredGraph graph, UnsafeNodeConstructor nodeConstructor) {
800             return nodeConstructor.create(ConstantNode.forLong(0L, graph), OFF_HEAP_LOCATION);
801         }
802 
isLoad(ValueNode node)803         private static boolean isLoad(ValueNode node) {
804             return node.getStackKind() != JavaKind.Void;
805         }
806 
setResult(ValueNode node, GraphBuilderContext b)807         private void setResult(ValueNode node, GraphBuilderContext b) {
808             if (isLoad(node)) {
809                 b.addPush(unsafeAccessKind, node);
810             } else {
811                 b.add(node);
812             }
813         }
814 
createUnsafeAccess(ValueNode value, GraphBuilderContext b, UnsafeNodeConstructor nodeConstructor)815         protected final void createUnsafeAccess(ValueNode value, GraphBuilderContext b, UnsafeNodeConstructor nodeConstructor) {
816             StructuredGraph graph = b.getGraph();
817             graph.markUnsafeAccess();
818             /* For unsafe access object pointers can only be stored in the heap */
819             if (unsafeAccessKind == JavaKind.Object) {
820                 setResult(createObjectAccessNode(value, nodeConstructor), b);
821             } else if (StampTool.isPointerAlwaysNull(value)) {
822                 setResult(createMemoryAccessNode(graph, nodeConstructor), b);
823             } else if (!explicitUnsafeNullChecks || StampTool.isPointerNonNull(value)) {
824                 setResult(createObjectAccessNode(value, nodeConstructor), b);
825             } else {
826                 FixedWithNextNode objectAccess = graph.add(createObjectAccessNode(value, nodeConstructor));
827                 FixedWithNextNode memoryAccess = graph.add(createMemoryAccessNode(graph, nodeConstructor));
828                 FixedWithNextNode[] accessNodes = new FixedWithNextNode[]{objectAccess, memoryAccess};
829 
830                 LogicNode condition = graph.addOrUniqueWithInputs(IsNullNode.create(value));
831                 b.add(new IfNode(condition, memoryAccess, objectAccess, 0.5));
832 
833                 MergeNode merge = b.append(new MergeNode());
834                 for (FixedWithNextNode node : accessNodes) {
835                     EndNode endNode = graph.add(new EndNode());
836                     node.setNext(endNode);
837                     if (node instanceof StateSplit) {
838                         b.setStateAfter((StateSplit) node);
839                     }
840                     merge.addForwardEnd(endNode);
841                 }
842 
843                 if (isLoad(objectAccess)) {
844                     ValuePhiNode phi = new ValuePhiNode(objectAccess.stamp(NodeView.DEFAULT), merge, accessNodes);
845                     b.push(unsafeAccessKind, graph.addOrUnique(phi));
846                 }
847                 b.setStateAfter(merge);
848             }
849         }
850     }
851 
852     public static class UnsafeGetPlugin extends UnsafeAccessPlugin {
853         private final AccessKind accessKind;
854 
UnsafeGetPlugin(JavaKind returnKind, boolean explicitUnsafeNullChecks)855         public UnsafeGetPlugin(JavaKind returnKind, boolean explicitUnsafeNullChecks) {
856             this(returnKind, AccessKind.PLAIN, explicitUnsafeNullChecks);
857         }
858 
UnsafeGetPlugin(JavaKind kind, AccessKind accessKind, boolean explicitUnsafeNullChecks)859         public UnsafeGetPlugin(JavaKind kind, AccessKind accessKind, boolean explicitUnsafeNullChecks) {
860             super(kind, explicitUnsafeNullChecks);
861             this.accessKind = accessKind;
862         }
863 
864         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode address)865         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode address) {
866             // Emits a null-check for the otherwise unused receiver
867             unsafe.get();
868             b.addPush(unsafeAccessKind, new UnsafeMemoryLoadNode(address, unsafeAccessKind, OFF_HEAP_LOCATION));
869             b.getGraph().markUnsafeAccess();
870             return true;
871         }
872 
873         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset)874         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset) {
875             // Emits a null-check for the otherwise unused receiver
876             unsafe.get();
877             if (accessKind.emitBarriers) {
878                 b.add(new MembarNode(accessKind.preReadBarriers));
879             }
880             createUnsafeAccess(object, b, (obj, loc) -> new RawLoadNode(obj, offset, unsafeAccessKind, loc));
881             if (accessKind.emitBarriers) {
882                 b.add(new MembarNode(accessKind.postReadBarriers));
883             }
884             return true;
885         }
886     }
887 
888     public static class UnsafePutPlugin extends UnsafeAccessPlugin {
889         private final AccessKind accessKind;
890 
UnsafePutPlugin(JavaKind kind, boolean explicitUnsafeNullChecks)891         public UnsafePutPlugin(JavaKind kind, boolean explicitUnsafeNullChecks) {
892             this(kind, AccessKind.PLAIN, explicitUnsafeNullChecks);
893         }
894 
UnsafePutPlugin(JavaKind kind, AccessKind accessKind, boolean explicitUnsafeNullChecks)895         private UnsafePutPlugin(JavaKind kind, AccessKind accessKind, boolean explicitUnsafeNullChecks) {
896             super(kind, explicitUnsafeNullChecks);
897             this.accessKind = accessKind;
898         }
899 
900         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode address, ValueNode value)901         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode address, ValueNode value) {
902             assert !accessKind.emitBarriers : "Barriers for address based Unsafe put is not supported.";
903             // Emits a null-check for the otherwise unused receiver
904             unsafe.get();
905             b.add(new UnsafeMemoryStoreNode(address, value, unsafeAccessKind, OFF_HEAP_LOCATION));
906             b.getGraph().markUnsafeAccess();
907             return true;
908         }
909 
910         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode value)911         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode value) {
912             // Emits a null-check for the otherwise unused receiver
913             unsafe.get();
914             if (accessKind.emitBarriers) {
915                 b.add(new MembarNode(accessKind.preWriteBarriers));
916             }
917             ValueNode maskedValue = b.maskSubWordValue(value, unsafeAccessKind);
918             createUnsafeAccess(object, b, (obj, loc) -> new RawStoreNode(obj, offset, maskedValue, unsafeAccessKind, loc));
919             if (accessKind.emitBarriers) {
920                 b.add(new MembarNode(accessKind.postWriteBarriers));
921             }
922             return true;
923         }
924     }
925 
926     public static class UnsafeFencePlugin implements InvocationPlugin {
927 
928         private final int barriers;
929 
UnsafeFencePlugin(int barriers)930         public UnsafeFencePlugin(int barriers) {
931             this.barriers = barriers;
932         }
933 
934         @Override
apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe)935         public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe) {
936             // Emits a null-check for the otherwise unused receiver
937             unsafe.get();
938             b.add(new MembarNode(barriers));
939             return true;
940         }
941     }
942 
943     private static final class DirectiveSpeculationReason implements SpeculationLog.SpeculationReason {
944         private final BytecodePosition pos;
945 
DirectiveSpeculationReason(BytecodePosition pos)946         private DirectiveSpeculationReason(BytecodePosition pos) {
947             this.pos = pos;
948         }
949 
950         @Override
hashCode()951         public int hashCode() {
952             return pos.hashCode();
953         }
954 
955         @Override
equals(Object obj)956         public boolean equals(Object obj) {
957             return obj instanceof DirectiveSpeculationReason && ((DirectiveSpeculationReason) obj).pos.equals(this.pos);
958         }
959     }
960 
registerGraalDirectivesPlugins(InvocationPlugins plugins)961     private static void registerGraalDirectivesPlugins(InvocationPlugins plugins) {
962         Registration r = new Registration(plugins, GraalDirectives.class);
963         r.register0("deoptimize", new InvocationPlugin() {
964             @Override
965             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
966                 b.add(new DeoptimizeNode(DeoptimizationAction.None, DeoptimizationReason.TransferToInterpreter));
967                 return true;
968             }
969         });
970 
971         r.register0("deoptimizeAndInvalidate", new InvocationPlugin() {
972             @Override
973             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
974                 b.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TransferToInterpreter));
975                 return true;
976             }
977         });
978 
979         r.register0("deoptimizeAndInvalidateWithSpeculation", new InvocationPlugin() {
980             @Override
981             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
982                 GraalError.guarantee(b.getGraph().getSpeculationLog() != null, "A speculation log is need to use `deoptimizeAndInvalidateWithSpeculation`");
983                 BytecodePosition pos = new BytecodePosition(null, b.getMethod(), b.bci());
984                 DirectiveSpeculationReason reason = new DirectiveSpeculationReason(pos);
985                 Speculation speculation;
986                 if (b.getGraph().getSpeculationLog().maySpeculate(reason)) {
987                     speculation = b.getGraph().getSpeculationLog().speculate(reason);
988                 } else {
989                     speculation = SpeculationLog.NO_SPECULATION;
990                 }
991                 b.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TransferToInterpreter, speculation));
992                 return true;
993             }
994         });
995 
996         r.register0("inCompiledCode", new InvocationPlugin() {
997             @Override
998             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
999                 b.addPush(JavaKind.Boolean, ConstantNode.forBoolean(true));
1000                 return true;
1001             }
1002         });
1003 
1004         r.register0("controlFlowAnchor", new InvocationPlugin() {
1005             @Override
1006             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
1007                 b.add(new ControlFlowAnchorNode());
1008                 return true;
1009             }
1010         });
1011 
1012         r.register2("injectBranchProbability", double.class, boolean.class, new InvocationPlugin() {
1013             @Override
1014             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode probability, ValueNode condition) {
1015                 b.addPush(JavaKind.Boolean, new BranchProbabilityNode(probability, condition));
1016                 return true;
1017             }
1018         });
1019 
1020         InvocationPlugin blackholePlugin = new InvocationPlugin() {
1021             @Override
1022             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
1023                 b.add(new BlackholeNode(value));
1024                 return true;
1025             }
1026         };
1027 
1028         InvocationPlugin bindToRegisterPlugin = new InvocationPlugin() {
1029             @Override
1030             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
1031                 b.add(new BindToRegisterNode(value));
1032                 return true;
1033             }
1034         };
1035         for (JavaKind kind : JavaKind.values()) {
1036             if ((kind.isPrimitive() && kind != JavaKind.Void) || kind == JavaKind.Object) {
1037                 Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
1038                 r.register1("blackhole", javaClass, blackholePlugin);
1039                 r.register1("bindToRegister", javaClass, bindToRegisterPlugin);
1040 
1041                 r.register1("opaque", javaClass, new InvocationPlugin() {
1042                     @Override
1043                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
1044                         b.addPush(kind, new OpaqueNode(value));
1045                         return true;
1046                     }
1047                 });
1048             }
1049         }
1050 
1051         InvocationPlugin spillPlugin = new InvocationPlugin() {
1052             @Override
1053             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
1054                 b.add(new SpillRegistersNode());
1055                 return true;
1056             }
1057         };
1058         r.register0("spillRegisters", spillPlugin);
1059 
1060         r.register1("guardingNonNull", Object.class, new InvocationPlugin() {
1061             @Override
1062             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
1063                 b.addPush(value.getStackKind(), b.nullCheckedValue(value));
1064                 return true;
1065             }
1066         });
1067 
1068         r.register1("ensureVirtualized", Object.class, new InvocationPlugin() {
1069             @Override
1070             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode object) {
1071                 b.add(new EnsureVirtualizedNode(object, false));
1072                 return true;
1073             }
1074         });
1075         r.register1("ensureVirtualizedHere", Object.class, new InvocationPlugin() {
1076             @Override
1077             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode object) {
1078                 b.add(new EnsureVirtualizedNode(object, true));
1079                 return true;
1080             }
1081         });
1082     }
1083 
registerJMHBlackholePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider)1084     private static void registerJMHBlackholePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
1085         InvocationPlugin blackholePlugin = new InvocationPlugin() {
1086             @Override
1087             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver blackhole, ValueNode value) {
1088                 blackhole.get();
1089                 b.add(new BlackholeNode(value));
1090                 return true;
1091             }
1092 
1093             @Override
1094             public boolean isDecorator() {
1095                 return true;
1096             }
1097         };
1098         String[] names = {"org.openjdk.jmh.infra.Blackhole", "org.openjdk.jmh.logic.BlackHole"};
1099         for (String name : names) {
1100             Registration r = new Registration(plugins, name, bytecodeProvider);
1101             for (JavaKind kind : JavaKind.values()) {
1102                 if ((kind.isPrimitive() && kind != JavaKind.Void) || kind == JavaKind.Object) {
1103                     Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
1104                     r.registerOptional2("consume", Receiver.class, javaClass, blackholePlugin);
1105                 }
1106             }
1107             r.registerOptional2("consume", Receiver.class, Object[].class, blackholePlugin);
1108         }
1109     }
1110 
registerJFRThrowablePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider)1111     private static void registerJFRThrowablePlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
1112         Registration r = new Registration(plugins, "oracle.jrockit.jfr.jdkevents.ThrowableTracer", bytecodeProvider);
1113         r.register2("traceThrowable", Throwable.class, String.class, new InvocationPlugin() {
1114             @Override
1115             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode throwable, ValueNode message) {
1116                 b.add(new VirtualizableInvokeMacroNode(b.getInvokeKind(), targetMethod, b.bci(), b.getInvokeReturnStamp(b.getAssumptions()), throwable, message));
1117                 return true;
1118             }
1119 
1120             @Override
1121             public boolean inlineOnly() {
1122                 return true;
1123             }
1124         });
1125     }
1126 
registerMethodHandleImplPlugins(InvocationPlugins plugins, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider)1127     private static void registerMethodHandleImplPlugins(InvocationPlugins plugins, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) {
1128         Registration r = new Registration(plugins, "java.lang.invoke.MethodHandleImpl", bytecodeProvider);
1129         r.register2("profileBoolean", boolean.class, int[].class, new InvocationPlugin() {
1130             @Override
1131             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode result, ValueNode counters) {
1132                 if (result.isConstant()) {
1133                     b.push(JavaKind.Boolean, result);
1134                     return true;
1135                 }
1136                 if (counters.isConstant()) {
1137                     ValueNode newResult = result;
1138                     int[] ctrs = snippetReflection.asObject(int[].class, (JavaConstant) counters.asConstant());
1139                     if (ctrs != null && ctrs.length == 2) {
1140                         int falseCount = ctrs[0];
1141                         int trueCount = ctrs[1];
1142                         int totalCount = trueCount + falseCount;
1143 
1144                         if (totalCount == 0) {
1145                             b.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TransferToInterpreter));
1146                         } else if (falseCount == 0 || trueCount == 0) {
1147                             boolean expected = falseCount == 0;
1148                             LogicNode condition = b.addWithInputs(
1149                                             IntegerEqualsNode.create(b.getConstantReflection(), b.getMetaAccess(), b.getOptions(), null, result, b.add(ConstantNode.forBoolean(!expected)),
1150                                                             NodeView.DEFAULT));
1151                             b.append(new FixedGuardNode(condition, DeoptimizationReason.UnreachedCode, DeoptimizationAction.InvalidateReprofile, true));
1152                             newResult = b.add(ConstantNode.forBoolean(expected));
1153                         } else {
1154                             // We cannot use BranchProbabilityNode here since there's no guarantee
1155                             // the result of MethodHandleImpl.profileBoolean() is used as the
1156                             // test in an `if` statement (as required by BranchProbabilityNode).
1157                         }
1158                     }
1159                     b.addPush(JavaKind.Boolean, newResult);
1160                     return true;
1161                 }
1162                 return false;
1163             }
1164         });
1165     }
1166 
1167     /**
1168      * Registers a plugin to ignore {@code com.sun.tdk.jcov.runtime.Collect.hit} within an
1169      * intrinsic.
1170      */
registerJcovCollectPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider)1171     private static void registerJcovCollectPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
1172         Registration r = new Registration(plugins, "com.sun.tdk.jcov.runtime.Collect", bytecodeProvider);
1173         r.register1("hit", int.class, new InvocationPlugin() {
1174             @Override
1175             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode object) {
1176                 if (b.parsingIntrinsic()) {
1177                     return true;
1178                 }
1179                 return false;
1180             }
1181         });
1182     }
1183 }
1184