1 /*
2  * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.
8  *
9  * This code is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * version 2 for more details (a copy is included in the LICENSE file that
13  * accompanied this code).
14  *
15  * You should have received a copy of the GNU General Public License version
16  * 2 along with this work; if not, write to the Free Software Foundation,
17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20  * or visit www.oracle.com if you need additional information or have any
21  * questions.
22  */
23 
24 
25 package org.graalvm.compiler.replacements.amd64;
26 
27 import static org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.registerPlatformSpecificUnsafePlugins;
28 import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW;
29 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.COS;
30 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.EXP;
31 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG;
32 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10;
33 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN;
34 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN;
35 
36 import java.util.Arrays;
37 
38 import org.graalvm.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool.RoundingMode;
39 import org.graalvm.compiler.nodes.PauseNode;
40 import org.graalvm.compiler.nodes.ValueNode;
41 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
42 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
43 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
44 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver;
45 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
46 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
47 import org.graalvm.compiler.nodes.java.AtomicReadAndAddNode;
48 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode;
49 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
50 import org.graalvm.compiler.nodes.spi.Replacements;
51 import org.graalvm.compiler.replacements.ArraysSubstitutions;
52 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeAccessPlugin;
53 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeGetPlugin;
54 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafePutPlugin;
55 import org.graalvm.compiler.replacements.TargetGraphBuilderPlugins;
56 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode;
57 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation;
58 import org.graalvm.compiler.replacements.nodes.BitCountNode;
59 import org.graalvm.compiler.replacements.nodes.FusedMultiplyAddNode;
60 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode;
61 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation;
62 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
63 
64 import jdk.vm.ci.amd64.AMD64;
65 import jdk.vm.ci.amd64.AMD64.CPUFeature;
66 import jdk.vm.ci.code.Architecture;
67 import jdk.vm.ci.meta.JavaKind;
68 import jdk.vm.ci.meta.ResolvedJavaMethod;
69 import sun.misc.Unsafe;
70 
71 public class AMD64GraphBuilderPlugins implements TargetGraphBuilderPlugins {
72     @Override
register(Plugins plugins, Replacements replacements, Architecture architecture, boolean explicitUnsafeNullChecks, boolean registerForeignCallMath, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics)73     public void register(Plugins plugins, Replacements replacements, Architecture architecture, boolean explicitUnsafeNullChecks,
74                     boolean registerForeignCallMath, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics) {
75         register(plugins, replacements, (AMD64) architecture, explicitUnsafeNullChecks, emitJDK9StringSubstitutions, useFMAIntrinsics);
76     }
77 
register(Plugins plugins, Replacements replacements, AMD64 arch, boolean explicitUnsafeNullChecks, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics)78     public static void register(Plugins plugins, Replacements replacements, AMD64 arch, boolean explicitUnsafeNullChecks,
79                     boolean emitJDK9StringSubstitutions,
80                     boolean useFMAIntrinsics) {
81         InvocationPlugins invocationPlugins = plugins.getInvocationPlugins();
82         invocationPlugins.defer(new Runnable() {
83             @Override
84             public void run() {
85                 registerThreadPlugins(invocationPlugins, arch);
86                 registerIntegerLongPlugins(invocationPlugins, AMD64IntegerSubstitutions.class, JavaKind.Int, arch, replacements);
87                 registerIntegerLongPlugins(invocationPlugins, AMD64LongSubstitutions.class, JavaKind.Long, arch, replacements);
88                 registerPlatformSpecificUnsafePlugins(invocationPlugins, replacements, explicitUnsafeNullChecks,
89                                 new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object, JavaKind.Boolean, JavaKind.Byte, JavaKind.Short, JavaKind.Char, JavaKind.Float, JavaKind.Double});
90                 registerUnsafePlugins(invocationPlugins, replacements, explicitUnsafeNullChecks);
91                 registerStringPlugins(invocationPlugins, replacements);
92                 if (emitJDK9StringSubstitutions) {
93                     registerStringLatin1Plugins(invocationPlugins, replacements);
94                     registerStringUTF16Plugins(invocationPlugins, replacements);
95                 }
96                 registerMathPlugins(invocationPlugins, useFMAIntrinsics, arch, replacements);
97                 registerArraysEqualsPlugins(invocationPlugins, replacements);
98             }
99         });
100     }
101 
registerThreadPlugins(InvocationPlugins plugins, AMD64 arch)102     private static void registerThreadPlugins(InvocationPlugins plugins, AMD64 arch) {
103         if (JavaVersionUtil.JAVA_SPEC > 8) {
104             // Pause instruction introduced with SSE2
105             assert (arch.getFeatures().contains(AMD64.CPUFeature.SSE2));
106             Registration r = new Registration(plugins, Thread.class);
107             r.register0("onSpinWait", new InvocationPlugin() {
108                 @Override
109                 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
110                     b.append(new PauseNode());
111                     return true;
112                 }
113             });
114         }
115     }
116 
registerIntegerLongPlugins(InvocationPlugins plugins, Class<?> substituteDeclaringClass, JavaKind kind, AMD64 arch, Replacements replacements)117     private static void registerIntegerLongPlugins(InvocationPlugins plugins, Class<?> substituteDeclaringClass, JavaKind kind, AMD64 arch, Replacements replacements) {
118         Class<?> declaringClass = kind.toBoxedJavaClass();
119         Class<?> type = kind.toJavaClass();
120         Registration r = new Registration(plugins, declaringClass, replacements);
121         r.registerMethodSubstitution(substituteDeclaringClass, "numberOfLeadingZeros", type);
122         r.registerMethodSubstitution(substituteDeclaringClass, "numberOfTrailingZeros", type);
123 
124         r.registerConditional1(arch.getFeatures().contains(AMD64.CPUFeature.POPCNT),
125                         "bitCount", type, new InvocationPlugin() {
126                             @Override
127                             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
128                                 b.push(JavaKind.Int, b.append(new BitCountNode(value).canonical(null)));
129                                 return true;
130                             }
131                         });
132     }
133 
registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, Replacements replacements)134     private static void registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, Replacements replacements) {
135         Registration r = new Registration(plugins, Math.class, replacements);
136         registerUnaryMath(r, "log", LOG);
137         registerUnaryMath(r, "log10", LOG10);
138         registerUnaryMath(r, "exp", EXP);
139         registerBinaryMath(r, "pow", POW);
140         registerUnaryMath(r, "sin", SIN);
141         registerUnaryMath(r, "cos", COS);
142         registerUnaryMath(r, "tan", TAN);
143 
144         boolean roundEnabled = arch.getFeatures().contains(CPUFeature.SSE4_1);
145         registerRound(roundEnabled, r, "rint", RoundingMode.NEAREST);
146         registerRound(roundEnabled, r, "ceil", RoundingMode.UP);
147         registerRound(roundEnabled, r, "floor", RoundingMode.DOWN);
148 
149         if (JavaVersionUtil.JAVA_SPEC > 8) {
150             registerFMA(r, useFMAIntrinsics && arch.getFeatures().contains(CPUFeature.FMA));
151         }
152     }
153 
registerFMA(Registration r, boolean isEnabled)154     private static void registerFMA(Registration r, boolean isEnabled) {
155         r.registerConditional3(isEnabled, "fma",
156                         Double.TYPE,
157                         Double.TYPE,
158                         Double.TYPE,
159                         new InvocationPlugin() {
160                             @Override
161                             public boolean apply(GraphBuilderContext b,
162                                             ResolvedJavaMethod targetMethod,
163                                             Receiver receiver,
164                                             ValueNode na,
165                                             ValueNode nb,
166                                             ValueNode nc) {
167                                 b.push(JavaKind.Double, b.append(new FusedMultiplyAddNode(na, nb, nc)));
168                                 return true;
169                             }
170                         });
171         r.registerConditional3(isEnabled, "fma",
172                         Float.TYPE,
173                         Float.TYPE,
174                         Float.TYPE,
175                         new InvocationPlugin() {
176                             @Override
177                             public boolean apply(GraphBuilderContext b,
178                                             ResolvedJavaMethod targetMethod,
179                                             Receiver receiver,
180                                             ValueNode na,
181                                             ValueNode nb,
182                                             ValueNode nc) {
183                                 b.push(JavaKind.Float, b.append(new FusedMultiplyAddNode(na, nb, nc)));
184                                 return true;
185                             }
186                         });
187     }
188 
registerUnaryMath(Registration r, String name, UnaryOperation operation)189     private static void registerUnaryMath(Registration r, String name, UnaryOperation operation) {
190         r.register1(name, Double.TYPE, new InvocationPlugin() {
191             @Override
192             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
193                 b.push(JavaKind.Double, b.append(UnaryMathIntrinsicNode.create(value, operation)));
194                 return true;
195             }
196         });
197     }
198 
registerBinaryMath(Registration r, String name, BinaryOperation operation)199     private static void registerBinaryMath(Registration r, String name, BinaryOperation operation) {
200         r.register2(name, Double.TYPE, Double.TYPE, new InvocationPlugin() {
201             @Override
202             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
203                 b.push(JavaKind.Double, b.append(BinaryMathIntrinsicNode.create(x, y, operation)));
204                 return true;
205             }
206         });
207     }
208 
registerRound(boolean isEnabled, Registration r, String name, RoundingMode mode)209     private static void registerRound(boolean isEnabled, Registration r, String name, RoundingMode mode) {
210         r.registerConditional1(isEnabled, name, Double.TYPE, new InvocationPlugin() {
211             @Override
212             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) {
213                 b.push(JavaKind.Double, b.append(new AMD64RoundNode(arg, mode)));
214                 return true;
215             }
216         });
217     }
218 
registerStringPlugins(InvocationPlugins plugins, Replacements replacements)219     private static void registerStringPlugins(InvocationPlugins plugins, Replacements replacements) {
220         if (JavaVersionUtil.JAVA_SPEC <= 8) {
221             Registration r;
222             r = new Registration(plugins, String.class, replacements);
223             r.setAllowOverwrite(true);
224             r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class,
225                             int.class, char[].class, int.class, int.class, int.class);
226             r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class);
227             r.registerMethodSubstitution(AMD64StringSubstitutions.class, "compareTo", Receiver.class, String.class);
228         }
229     }
230 
registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements)231     private static void registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements) {
232         Registration r = new Registration(plugins, "java.lang.StringLatin1", replacements);
233         r.setAllowOverwrite(true);
234         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareTo", byte[].class, byte[].class);
235         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareToUTF16", byte[].class, byte[].class);
236         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, char[].class, int.class, int.class);
237         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, byte[].class, int.class, int.class);
238         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class);
239         r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, byte[].class, int.class, int.class);
240     }
241 
registerStringUTF16Plugins(InvocationPlugins plugins, Replacements replacements)242     private static void registerStringUTF16Plugins(InvocationPlugins plugins, Replacements replacements) {
243         Registration r = new Registration(plugins, "java.lang.StringUTF16", replacements);
244         r.setAllowOverwrite(true);
245         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareTo", byte[].class, byte[].class);
246         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareToLatin1", byte[].class, byte[].class);
247         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", char[].class, int.class, byte[].class, int.class, int.class);
248         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", byte[].class, int.class, byte[].class, int.class, int.class);
249         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class);
250         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class);
251         r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfLatin1Unsafe", byte[].class, int.class, byte[].class, int.class, int.class);
252     }
253 
registerUnsafePlugins(InvocationPlugins plugins, Replacements replacements, boolean explicitUnsafeNullChecks)254     private static void registerUnsafePlugins(InvocationPlugins plugins, Replacements replacements, boolean explicitUnsafeNullChecks) {
255         registerUnsafePlugins(new Registration(plugins, Unsafe.class), explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object}, true);
256         if (JavaVersionUtil.JAVA_SPEC > 8) {
257             registerUnsafePlugins(new Registration(plugins, "jdk.internal.misc.Unsafe", replacements), explicitUnsafeNullChecks,
258                             new JavaKind[]{JavaKind.Boolean, JavaKind.Byte, JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long, JavaKind.Object},
259                             JavaVersionUtil.JAVA_SPEC <= 11);
260         }
261     }
262 
registerUnsafePlugins(Registration r, boolean explicitUnsafeNullChecks, JavaKind[] unsafeJavaKinds, boolean java11OrEarlier)263     private static void registerUnsafePlugins(Registration r, boolean explicitUnsafeNullChecks, JavaKind[] unsafeJavaKinds, boolean java11OrEarlier) {
264         for (JavaKind kind : unsafeJavaKinds) {
265             Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
266             String kindName = (kind == JavaKind.Object && !java11OrEarlier) ? "Reference" : kind.name();
267             r.register4("getAndSet" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) {
268                 @Override
269                 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode value) {
270                     // Emits a null-check for the otherwise unused receiver
271                     unsafe.get();
272                     createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndWriteNode(obj, offset, value, kind, loc));
273                     return true;
274                 }
275             });
276             if (kind != JavaKind.Boolean && kind.isNumericInteger()) {
277                 r.register4("getAndAdd" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) {
278                     @Override
279                     public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode delta) {
280                         // Emits a null-check for the otherwise unused receiver
281                         unsafe.get();
282                         createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndAddNode(b.add(new OffsetAddressNode(obj, offset)), delta, kind, loc));
283                         return true;
284                     }
285                 });
286             }
287         }
288 
289         for (JavaKind kind : new JavaKind[]{JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long}) {
290             Class<?> javaClass = kind.toJavaClass();
291             r.registerOptional3("get" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, explicitUnsafeNullChecks));
292             r.registerOptional4("put" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, explicitUnsafeNullChecks));
293         }
294     }
295 
registerArraysEqualsPlugins(InvocationPlugins plugins, Replacements replacements)296     private static void registerArraysEqualsPlugins(InvocationPlugins plugins, Replacements replacements) {
297         Registration r = new Registration(plugins, Arrays.class, replacements);
298         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", float[].class, float[].class);
299         r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", double[].class, double[].class);
300     }
301 }
302