1 /* 2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements.amd64; 26 27 import static org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.registerPlatformSpecificUnsafePlugins; 28 import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW; 29 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.COS; 30 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.EXP; 31 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG; 32 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; 33 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; 34 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; 35 36 import java.util.Arrays; 37 38 import org.graalvm.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool.RoundingMode; 39 import org.graalvm.compiler.nodes.PauseNode; 40 import org.graalvm.compiler.nodes.ValueNode; 41 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; 42 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; 43 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; 44 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver; 45 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; 46 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration; 47 import org.graalvm.compiler.nodes.java.AtomicReadAndAddNode; 48 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode; 49 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; 50 import org.graalvm.compiler.nodes.spi.Replacements; 51 import org.graalvm.compiler.replacements.ArraysSubstitutions; 52 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeAccessPlugin; 53 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeGetPlugin; 54 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafePutPlugin; 55 import org.graalvm.compiler.replacements.TargetGraphBuilderPlugins; 56 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; 57 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; 58 import org.graalvm.compiler.replacements.nodes.BitCountNode; 59 import org.graalvm.compiler.replacements.nodes.FusedMultiplyAddNode; 60 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; 61 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; 62 import org.graalvm.compiler.serviceprovider.JavaVersionUtil; 63 64 import jdk.vm.ci.amd64.AMD64; 65 import jdk.vm.ci.amd64.AMD64.CPUFeature; 66 import jdk.vm.ci.code.Architecture; 67 import jdk.vm.ci.meta.JavaKind; 68 import jdk.vm.ci.meta.ResolvedJavaMethod; 69 import sun.misc.Unsafe; 70 71 public class AMD64GraphBuilderPlugins implements TargetGraphBuilderPlugins { 72 @Override register(Plugins plugins, Replacements replacements, Architecture architecture, boolean explicitUnsafeNullChecks, boolean registerForeignCallMath, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics)73 public void register(Plugins plugins, Replacements replacements, Architecture architecture, boolean explicitUnsafeNullChecks, 74 boolean registerForeignCallMath, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics) { 75 register(plugins, replacements, (AMD64) architecture, explicitUnsafeNullChecks, emitJDK9StringSubstitutions, useFMAIntrinsics); 76 } 77 register(Plugins plugins, Replacements replacements, AMD64 arch, boolean explicitUnsafeNullChecks, boolean emitJDK9StringSubstitutions, boolean useFMAIntrinsics)78 public static void register(Plugins plugins, Replacements replacements, AMD64 arch, boolean explicitUnsafeNullChecks, 79 boolean emitJDK9StringSubstitutions, 80 boolean useFMAIntrinsics) { 81 InvocationPlugins invocationPlugins = plugins.getInvocationPlugins(); 82 invocationPlugins.defer(new Runnable() { 83 @Override 84 public void run() { 85 registerThreadPlugins(invocationPlugins, arch); 86 registerIntegerLongPlugins(invocationPlugins, AMD64IntegerSubstitutions.class, JavaKind.Int, arch, replacements); 87 registerIntegerLongPlugins(invocationPlugins, AMD64LongSubstitutions.class, JavaKind.Long, arch, replacements); 88 registerPlatformSpecificUnsafePlugins(invocationPlugins, replacements, explicitUnsafeNullChecks, 89 new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object, JavaKind.Boolean, JavaKind.Byte, JavaKind.Short, JavaKind.Char, JavaKind.Float, JavaKind.Double}); 90 registerUnsafePlugins(invocationPlugins, replacements, explicitUnsafeNullChecks); 91 registerStringPlugins(invocationPlugins, replacements); 92 if (emitJDK9StringSubstitutions) { 93 registerStringLatin1Plugins(invocationPlugins, replacements); 94 registerStringUTF16Plugins(invocationPlugins, replacements); 95 } 96 registerMathPlugins(invocationPlugins, useFMAIntrinsics, arch, replacements); 97 registerArraysEqualsPlugins(invocationPlugins, replacements); 98 } 99 }); 100 } 101 registerThreadPlugins(InvocationPlugins plugins, AMD64 arch)102 private static void registerThreadPlugins(InvocationPlugins plugins, AMD64 arch) { 103 if (JavaVersionUtil.JAVA_SPEC > 8) { 104 // Pause instruction introduced with SSE2 105 assert (arch.getFeatures().contains(AMD64.CPUFeature.SSE2)); 106 Registration r = new Registration(plugins, Thread.class); 107 r.register0("onSpinWait", new InvocationPlugin() { 108 @Override 109 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { 110 b.append(new PauseNode()); 111 return true; 112 } 113 }); 114 } 115 } 116 registerIntegerLongPlugins(InvocationPlugins plugins, Class<?> substituteDeclaringClass, JavaKind kind, AMD64 arch, Replacements replacements)117 private static void registerIntegerLongPlugins(InvocationPlugins plugins, Class<?> substituteDeclaringClass, JavaKind kind, AMD64 arch, Replacements replacements) { 118 Class<?> declaringClass = kind.toBoxedJavaClass(); 119 Class<?> type = kind.toJavaClass(); 120 Registration r = new Registration(plugins, declaringClass, replacements); 121 r.registerMethodSubstitution(substituteDeclaringClass, "numberOfLeadingZeros", type); 122 r.registerMethodSubstitution(substituteDeclaringClass, "numberOfTrailingZeros", type); 123 124 r.registerConditional1(arch.getFeatures().contains(AMD64.CPUFeature.POPCNT), 125 "bitCount", type, new InvocationPlugin() { 126 @Override 127 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { 128 b.push(JavaKind.Int, b.append(new BitCountNode(value).canonical(null))); 129 return true; 130 } 131 }); 132 } 133 registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, Replacements replacements)134 private static void registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, Replacements replacements) { 135 Registration r = new Registration(plugins, Math.class, replacements); 136 registerUnaryMath(r, "log", LOG); 137 registerUnaryMath(r, "log10", LOG10); 138 registerUnaryMath(r, "exp", EXP); 139 registerBinaryMath(r, "pow", POW); 140 registerUnaryMath(r, "sin", SIN); 141 registerUnaryMath(r, "cos", COS); 142 registerUnaryMath(r, "tan", TAN); 143 144 boolean roundEnabled = arch.getFeatures().contains(CPUFeature.SSE4_1); 145 registerRound(roundEnabled, r, "rint", RoundingMode.NEAREST); 146 registerRound(roundEnabled, r, "ceil", RoundingMode.UP); 147 registerRound(roundEnabled, r, "floor", RoundingMode.DOWN); 148 149 if (JavaVersionUtil.JAVA_SPEC > 8) { 150 registerFMA(r, useFMAIntrinsics && arch.getFeatures().contains(CPUFeature.FMA)); 151 } 152 } 153 registerFMA(Registration r, boolean isEnabled)154 private static void registerFMA(Registration r, boolean isEnabled) { 155 r.registerConditional3(isEnabled, "fma", 156 Double.TYPE, 157 Double.TYPE, 158 Double.TYPE, 159 new InvocationPlugin() { 160 @Override 161 public boolean apply(GraphBuilderContext b, 162 ResolvedJavaMethod targetMethod, 163 Receiver receiver, 164 ValueNode na, 165 ValueNode nb, 166 ValueNode nc) { 167 b.push(JavaKind.Double, b.append(new FusedMultiplyAddNode(na, nb, nc))); 168 return true; 169 } 170 }); 171 r.registerConditional3(isEnabled, "fma", 172 Float.TYPE, 173 Float.TYPE, 174 Float.TYPE, 175 new InvocationPlugin() { 176 @Override 177 public boolean apply(GraphBuilderContext b, 178 ResolvedJavaMethod targetMethod, 179 Receiver receiver, 180 ValueNode na, 181 ValueNode nb, 182 ValueNode nc) { 183 b.push(JavaKind.Float, b.append(new FusedMultiplyAddNode(na, nb, nc))); 184 return true; 185 } 186 }); 187 } 188 registerUnaryMath(Registration r, String name, UnaryOperation operation)189 private static void registerUnaryMath(Registration r, String name, UnaryOperation operation) { 190 r.register1(name, Double.TYPE, new InvocationPlugin() { 191 @Override 192 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { 193 b.push(JavaKind.Double, b.append(UnaryMathIntrinsicNode.create(value, operation))); 194 return true; 195 } 196 }); 197 } 198 registerBinaryMath(Registration r, String name, BinaryOperation operation)199 private static void registerBinaryMath(Registration r, String name, BinaryOperation operation) { 200 r.register2(name, Double.TYPE, Double.TYPE, new InvocationPlugin() { 201 @Override 202 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { 203 b.push(JavaKind.Double, b.append(BinaryMathIntrinsicNode.create(x, y, operation))); 204 return true; 205 } 206 }); 207 } 208 registerRound(boolean isEnabled, Registration r, String name, RoundingMode mode)209 private static void registerRound(boolean isEnabled, Registration r, String name, RoundingMode mode) { 210 r.registerConditional1(isEnabled, name, Double.TYPE, new InvocationPlugin() { 211 @Override 212 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) { 213 b.push(JavaKind.Double, b.append(new AMD64RoundNode(arg, mode))); 214 return true; 215 } 216 }); 217 } 218 registerStringPlugins(InvocationPlugins plugins, Replacements replacements)219 private static void registerStringPlugins(InvocationPlugins plugins, Replacements replacements) { 220 if (JavaVersionUtil.JAVA_SPEC <= 8) { 221 Registration r; 222 r = new Registration(plugins, String.class, replacements); 223 r.setAllowOverwrite(true); 224 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class, 225 int.class, char[].class, int.class, int.class, int.class); 226 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class); 227 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "compareTo", Receiver.class, String.class); 228 } 229 } 230 registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements)231 private static void registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements) { 232 Registration r = new Registration(plugins, "java.lang.StringLatin1", replacements); 233 r.setAllowOverwrite(true); 234 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareTo", byte[].class, byte[].class); 235 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareToUTF16", byte[].class, byte[].class); 236 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, char[].class, int.class, int.class); 237 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, byte[].class, int.class, int.class); 238 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class); 239 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, byte[].class, int.class, int.class); 240 } 241 registerStringUTF16Plugins(InvocationPlugins plugins, Replacements replacements)242 private static void registerStringUTF16Plugins(InvocationPlugins plugins, Replacements replacements) { 243 Registration r = new Registration(plugins, "java.lang.StringUTF16", replacements); 244 r.setAllowOverwrite(true); 245 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareTo", byte[].class, byte[].class); 246 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareToLatin1", byte[].class, byte[].class); 247 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", char[].class, int.class, byte[].class, int.class, int.class); 248 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", byte[].class, int.class, byte[].class, int.class, int.class); 249 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class); 250 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class); 251 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfLatin1Unsafe", byte[].class, int.class, byte[].class, int.class, int.class); 252 } 253 registerUnsafePlugins(InvocationPlugins plugins, Replacements replacements, boolean explicitUnsafeNullChecks)254 private static void registerUnsafePlugins(InvocationPlugins plugins, Replacements replacements, boolean explicitUnsafeNullChecks) { 255 registerUnsafePlugins(new Registration(plugins, Unsafe.class), explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object}, true); 256 if (JavaVersionUtil.JAVA_SPEC > 8) { 257 registerUnsafePlugins(new Registration(plugins, "jdk.internal.misc.Unsafe", replacements), explicitUnsafeNullChecks, 258 new JavaKind[]{JavaKind.Boolean, JavaKind.Byte, JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long, JavaKind.Object}, 259 JavaVersionUtil.JAVA_SPEC <= 11); 260 } 261 } 262 registerUnsafePlugins(Registration r, boolean explicitUnsafeNullChecks, JavaKind[] unsafeJavaKinds, boolean java11OrEarlier)263 private static void registerUnsafePlugins(Registration r, boolean explicitUnsafeNullChecks, JavaKind[] unsafeJavaKinds, boolean java11OrEarlier) { 264 for (JavaKind kind : unsafeJavaKinds) { 265 Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass(); 266 String kindName = (kind == JavaKind.Object && !java11OrEarlier) ? "Reference" : kind.name(); 267 r.register4("getAndSet" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) { 268 @Override 269 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode value) { 270 // Emits a null-check for the otherwise unused receiver 271 unsafe.get(); 272 createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndWriteNode(obj, offset, value, kind, loc)); 273 return true; 274 } 275 }); 276 if (kind != JavaKind.Boolean && kind.isNumericInteger()) { 277 r.register4("getAndAdd" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) { 278 @Override 279 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode delta) { 280 // Emits a null-check for the otherwise unused receiver 281 unsafe.get(); 282 createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndAddNode(b.add(new OffsetAddressNode(obj, offset)), delta, kind, loc)); 283 return true; 284 } 285 }); 286 } 287 } 288 289 for (JavaKind kind : new JavaKind[]{JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long}) { 290 Class<?> javaClass = kind.toJavaClass(); 291 r.registerOptional3("get" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, explicitUnsafeNullChecks)); 292 r.registerOptional4("put" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, explicitUnsafeNullChecks)); 293 } 294 } 295 registerArraysEqualsPlugins(InvocationPlugins plugins, Replacements replacements)296 private static void registerArraysEqualsPlugins(InvocationPlugins plugins, Replacements replacements) { 297 Registration r = new Registration(plugins, Arrays.class, replacements); 298 r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", float[].class, float[].class); 299 r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", double[].class, double[].class); 300 } 301 } 302