1 /* 2 * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.lir.amd64.vector; 26 27 import static jdk.vm.ci.code.ValueUtil.asRegister; 28 import static jdk.vm.ci.code.ValueUtil.isRegister; 29 import static jdk.vm.ci.code.ValueUtil.isStackSlot; 30 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVD; 31 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVDQU32; 32 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVQ; 33 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSD; 34 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSS; 35 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPD; 36 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPS; 37 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VXORPD; 38 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; 39 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; 40 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; 41 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; 42 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; 43 44 import org.graalvm.compiler.asm.amd64.AMD64Address; 45 import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp; 46 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; 47 import org.graalvm.compiler.asm.amd64.AVXKind; 48 import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; 49 import org.graalvm.compiler.debug.GraalError; 50 import org.graalvm.compiler.lir.LIRFrameState; 51 import org.graalvm.compiler.lir.LIRInstructionClass; 52 import org.graalvm.compiler.lir.Opcode; 53 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp; 54 import org.graalvm.compiler.lir.StandardOp.ValueMoveOp; 55 import org.graalvm.compiler.lir.amd64.AMD64AddressValue; 56 import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction; 57 import org.graalvm.compiler.lir.amd64.AMD64Move; 58 import org.graalvm.compiler.lir.amd64.AMD64RestoreRegistersOp; 59 import org.graalvm.compiler.lir.amd64.AMD64SaveRegistersOp; 60 import org.graalvm.compiler.lir.asm.CompilationResultBuilder; 61 62 import jdk.vm.ci.amd64.AMD64Kind; 63 import jdk.vm.ci.code.Register; 64 import jdk.vm.ci.code.RegisterValue; 65 import jdk.vm.ci.code.StackSlot; 66 import jdk.vm.ci.meta.AllocatableValue; 67 import jdk.vm.ci.meta.Constant; 68 import jdk.vm.ci.meta.JavaConstant; 69 import jdk.vm.ci.meta.Value; 70 71 public class AMD64VectorMove { 72 73 @Opcode("VMOVE") 74 public static final class MoveToRegOp extends AMD64LIRInstruction implements ValueMoveOp { 75 public static final LIRInstructionClass<MoveToRegOp> TYPE = LIRInstructionClass.create(MoveToRegOp.class); 76 77 @Def({REG, STACK, HINT}) protected AllocatableValue result; 78 @Use({REG, STACK}) protected AllocatableValue input; 79 MoveToRegOp(AllocatableValue result, AllocatableValue input)80 public MoveToRegOp(AllocatableValue result, AllocatableValue input) { 81 super(TYPE); 82 this.result = result; 83 this.input = input; 84 } 85 86 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)87 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 88 move(crb, masm, result, input); 89 } 90 91 @Override getInput()92 public AllocatableValue getInput() { 93 return input; 94 } 95 96 @Override getResult()97 public AllocatableValue getResult() { 98 return result; 99 } 100 } 101 102 @Opcode("VMOVE") 103 public static final class MoveFromRegOp extends AMD64LIRInstruction implements ValueMoveOp { 104 public static final LIRInstructionClass<MoveFromRegOp> TYPE = LIRInstructionClass.create(MoveFromRegOp.class); 105 106 @Def({REG, STACK}) protected AllocatableValue result; 107 @Use({REG, HINT}) protected AllocatableValue input; 108 MoveFromRegOp(AllocatableValue result, AllocatableValue input)109 public MoveFromRegOp(AllocatableValue result, AllocatableValue input) { 110 super(TYPE); 111 this.result = result; 112 this.input = input; 113 } 114 115 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)116 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 117 move(crb, masm, result, input); 118 } 119 120 @Override getInput()121 public AllocatableValue getInput() { 122 return input; 123 } 124 125 @Override getResult()126 public AllocatableValue getResult() { 127 return result; 128 } 129 } 130 131 @Opcode("VMOVE") 132 public static class MoveFromConstOp extends AMD64LIRInstruction implements LoadConstantOp { 133 public static final LIRInstructionClass<MoveFromConstOp> TYPE = LIRInstructionClass.create(MoveFromConstOp.class); 134 135 @Def({REG, STACK}) protected AllocatableValue result; 136 private final JavaConstant input; 137 MoveFromConstOp(AllocatableValue result, JavaConstant input)138 public MoveFromConstOp(AllocatableValue result, JavaConstant input) { 139 super(TYPE); 140 this.result = result; 141 this.input = input; 142 } 143 144 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)145 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 146 if (isRegister(result)) { 147 const2reg(crb, masm, (RegisterValue) result, input); 148 } else { 149 assert isStackSlot(result); 150 AMD64Move.const2stack(crb, masm, result, input); 151 } 152 } 153 154 @Override getConstant()155 public Constant getConstant() { 156 return input; 157 } 158 159 @Override getResult()160 public AllocatableValue getResult() { 161 return result; 162 } 163 } 164 165 @Opcode("VSTACKMOVE") 166 public static final class StackMoveOp extends AMD64LIRInstruction implements ValueMoveOp { 167 public static final LIRInstructionClass<StackMoveOp> TYPE = LIRInstructionClass.create(StackMoveOp.class); 168 169 @Def({STACK}) protected AllocatableValue result; 170 @Use({STACK, HINT}) protected AllocatableValue input; 171 @Alive({STACK, UNINITIALIZED}) private AllocatableValue backupSlot; 172 173 private Register scratch; 174 StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot)175 public StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot) { 176 super(TYPE); 177 this.result = result; 178 this.input = input; 179 this.backupSlot = backupSlot; 180 this.scratch = scratch; 181 } 182 183 @Override getInput()184 public AllocatableValue getInput() { 185 return input; 186 } 187 188 @Override getResult()189 public AllocatableValue getResult() { 190 return result; 191 } 192 193 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)194 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 195 // backup scratch register 196 move(crb, masm, backupSlot, scratch.asValue(backupSlot.getValueKind())); 197 // move stack slot 198 move(crb, masm, scratch.asValue(getInput().getValueKind()), getInput()); 199 move(crb, masm, getResult(), scratch.asValue(getResult().getValueKind())); 200 // restore scratch register 201 move(crb, masm, scratch.asValue(backupSlot.getValueKind()), backupSlot); 202 203 } 204 } 205 206 public abstract static class VectorMemOp extends AMD64VectorInstruction { 207 208 protected final VexMoveOp op; 209 210 @Use({COMPOSITE}) protected AMD64AddressValue address; 211 @State protected LIRFrameState state; 212 VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state)213 protected VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state) { 214 super(c, size); 215 this.op = op; 216 this.address = address; 217 this.state = state; 218 } 219 emitMemAccess(AMD64MacroAssembler masm)220 protected abstract void emitMemAccess(AMD64MacroAssembler masm); 221 222 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)223 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 224 if (state != null) { 225 crb.recordImplicitException(masm.position(), state); 226 } 227 emitMemAccess(masm); 228 } 229 } 230 231 public static final class VectorLoadOp extends VectorMemOp { 232 public static final LIRInstructionClass<VectorLoadOp> TYPE = LIRInstructionClass.create(VectorLoadOp.class); 233 234 @Def({REG}) protected AllocatableValue result; 235 VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state)236 public VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state) { 237 super(TYPE, size, op, address, state); 238 this.result = result; 239 } 240 241 @Override emitMemAccess(AMD64MacroAssembler masm)242 public void emitMemAccess(AMD64MacroAssembler masm) { 243 op.emit(masm, size, asRegister(result), address.toAddress()); 244 } 245 } 246 247 public static class VectorStoreOp extends VectorMemOp { 248 public static final LIRInstructionClass<VectorStoreOp> TYPE = LIRInstructionClass.create(VectorStoreOp.class); 249 250 @Use({REG}) protected AllocatableValue input; 251 VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state)252 public VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state) { 253 super(TYPE, size, op, address, state); 254 this.input = input; 255 } 256 257 @Override emitMemAccess(AMD64MacroAssembler masm)258 public void emitMemAccess(AMD64MacroAssembler masm) { 259 op.emit(masm, size, address.toAddress(), asRegister(input)); 260 } 261 } 262 263 @Opcode("SAVE_REGISTER") 264 public static class SaveRegistersOp extends AMD64SaveRegistersOp { 265 public static final LIRInstructionClass<SaveRegistersOp> TYPE = LIRInstructionClass.create(SaveRegistersOp.class); 266 SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots)267 public SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots) { 268 super(TYPE, savedRegisters, slots); 269 } 270 271 @Override saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register)272 protected void saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register) { 273 AMD64Kind kind = (AMD64Kind) result.getPlatformKind(); 274 if (kind.isXMM()) { 275 VexMoveOp op; 276 if (kind.getVectorLength() > 1) { 277 op = getVectorMoveOp(kind.getScalar()); 278 } else { 279 op = getScalarMoveOp(kind); 280 } 281 282 AMD64Address addr = (AMD64Address) crb.asAddress(result); 283 op.emit(masm, AVXKind.getRegisterSize(kind), addr, register); 284 } else { 285 super.saveRegister(crb, masm, result, register); 286 } 287 } 288 } 289 290 @Opcode("RESTORE_REGISTER") 291 public static final class RestoreRegistersOp extends AMD64RestoreRegistersOp { 292 public static final LIRInstructionClass<RestoreRegistersOp> TYPE = LIRInstructionClass.create(RestoreRegistersOp.class); 293 RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save)294 public RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save) { 295 super(TYPE, source, save); 296 } 297 298 @Override restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input)299 protected void restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input) { 300 AMD64Kind kind = (AMD64Kind) input.getPlatformKind(); 301 if (kind.isXMM()) { 302 VexMoveOp op; 303 if (kind.getVectorLength() > 1) { 304 op = getVectorMoveOp(kind.getScalar()); 305 } else { 306 op = getScalarMoveOp(kind); 307 } 308 309 AMD64Address addr = (AMD64Address) crb.asAddress(input); 310 op.emit(masm, AVXKind.getRegisterSize(kind), register, addr); 311 } else { 312 super.restoreRegister(crb, masm, register, input); 313 } 314 } 315 } 316 getScalarMoveOp(AMD64Kind kind)317 private static VexMoveOp getScalarMoveOp(AMD64Kind kind) { 318 switch (kind) { 319 case SINGLE: 320 return VMOVSS; 321 case DOUBLE: 322 return VMOVSD; 323 default: 324 throw GraalError.shouldNotReachHere(); 325 } 326 } 327 getVectorMoveOp(AMD64Kind kind)328 private static VexMoveOp getVectorMoveOp(AMD64Kind kind) { 329 switch (kind) { 330 case SINGLE: 331 return VMOVUPS; 332 case DOUBLE: 333 return VMOVUPD; 334 default: 335 return VMOVDQU32; 336 } 337 } 338 getVectorMemMoveOp(AMD64Kind kind)339 private static VexMoveOp getVectorMemMoveOp(AMD64Kind kind) { 340 switch (AVXKind.getDataSize(kind)) { 341 case DWORD: 342 return VMOVD; 343 case QWORD: 344 return VMOVQ; 345 default: 346 return getVectorMoveOp(kind.getScalar()); 347 } 348 } 349 move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input)350 private static void move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input) { 351 VexMoveOp op; 352 AVXSize size; 353 AMD64Kind kind = (AMD64Kind) result.getPlatformKind(); 354 if (kind.getVectorLength() > 1) { 355 size = AVXKind.getRegisterSize(kind); 356 if (isRegister(input) && isRegister(result)) { 357 op = getVectorMoveOp(kind.getScalar()); 358 } else { 359 op = getVectorMemMoveOp(kind); 360 } 361 } else { 362 size = AVXSize.XMM; 363 if (isRegister(input) && isRegister(result)) { 364 op = getVectorMoveOp(kind); 365 } else { 366 op = getScalarMoveOp(kind); 367 } 368 } 369 370 if (isRegister(input)) { 371 if (isRegister(result)) { 372 if (!asRegister(input).equals(asRegister(result))) { 373 op.emit(masm, size, asRegister(result), asRegister(input)); 374 } 375 } else { 376 assert isStackSlot(result); 377 op.emit(masm, size, (AMD64Address) crb.asAddress(result), asRegister(input)); 378 } 379 } else { 380 assert isStackSlot(input) && isRegister(result); 381 op.emit(masm, size, asRegister(result), (AMD64Address) crb.asAddress(input)); 382 } 383 } 384 const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input)385 private static void const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input) { 386 if (input.isDefaultForKind()) { 387 AMD64Kind kind = (AMD64Kind) result.getPlatformKind(); 388 Register register = result.getRegister(); 389 VXORPD.emit(masm, AVXKind.getRegisterSize(kind), register, register, register); 390 return; 391 } 392 393 AMD64Address address; 394 switch (input.getJavaKind()) { 395 case Float: 396 address = (AMD64Address) crb.asFloatConstRef(input); 397 break; 398 399 case Double: 400 address = (AMD64Address) crb.asDoubleConstRef(input); 401 break; 402 403 default: 404 throw GraalError.shouldNotReachHere(); 405 } 406 VexMoveOp op = getScalarMoveOp((AMD64Kind) result.getPlatformKind()); 407 op.emit(masm, AVXSize.XMM, asRegister(result), address); 408 } 409 410 public static final class AVXMoveToIntOp extends AMD64LIRInstruction { 411 public static final LIRInstructionClass<AVXMoveToIntOp> TYPE = LIRInstructionClass.create(AVXMoveToIntOp.class); 412 413 @Opcode private final VexMoveOp opcode; 414 415 @Def({REG, STACK}) protected AllocatableValue result; 416 @Use({REG}) protected AllocatableValue input; 417 AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input)418 public AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input) { 419 super(TYPE); 420 this.opcode = opcode; 421 this.result = result; 422 this.input = input; 423 } 424 425 @Override emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)426 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { 427 if (isRegister(result)) { 428 opcode.emitReverse(masm, AVXSize.XMM, asRegister(result), asRegister(input)); 429 } else { 430 opcode.emit(masm, AVXSize.XMM, (AMD64Address) crb.asAddress(result), asRegister(input)); 431 } 432 } 433 } 434 } 435