1 /*
2  * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.
8  *
9  * This code is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * version 2 for more details (a copy is included in the LICENSE file that
13  * accompanied this code).
14  *
15  * You should have received a copy of the GNU General Public License version
16  * 2 along with this work; if not, write to the Free Software Foundation,
17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20  * or visit www.oracle.com if you need additional information or have any
21  * questions.
22  */
23 
24 
25 package org.graalvm.compiler.lir.amd64.vector;
26 
27 import static jdk.vm.ci.code.ValueUtil.asRegister;
28 import static jdk.vm.ci.code.ValueUtil.isRegister;
29 import static jdk.vm.ci.code.ValueUtil.isStackSlot;
30 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVD;
31 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVDQU32;
32 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVQ;
33 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSD;
34 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSS;
35 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPD;
36 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPS;
37 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VXORPD;
38 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE;
39 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
40 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
41 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
42 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED;
43 
44 import org.graalvm.compiler.asm.amd64.AMD64Address;
45 import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp;
46 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
47 import org.graalvm.compiler.asm.amd64.AVXKind;
48 import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize;
49 import org.graalvm.compiler.debug.GraalError;
50 import org.graalvm.compiler.lir.LIRFrameState;
51 import org.graalvm.compiler.lir.LIRInstructionClass;
52 import org.graalvm.compiler.lir.Opcode;
53 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
54 import org.graalvm.compiler.lir.StandardOp.ValueMoveOp;
55 import org.graalvm.compiler.lir.amd64.AMD64AddressValue;
56 import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction;
57 import org.graalvm.compiler.lir.amd64.AMD64Move;
58 import org.graalvm.compiler.lir.amd64.AMD64RestoreRegistersOp;
59 import org.graalvm.compiler.lir.amd64.AMD64SaveRegistersOp;
60 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
61 
62 import jdk.vm.ci.amd64.AMD64Kind;
63 import jdk.vm.ci.code.Register;
64 import jdk.vm.ci.code.RegisterValue;
65 import jdk.vm.ci.code.StackSlot;
66 import jdk.vm.ci.meta.AllocatableValue;
67 import jdk.vm.ci.meta.Constant;
68 import jdk.vm.ci.meta.JavaConstant;
69 import jdk.vm.ci.meta.Value;
70 
71 public class AMD64VectorMove {
72 
73     @Opcode("VMOVE")
74     public static final class MoveToRegOp extends AMD64LIRInstruction implements ValueMoveOp {
75         public static final LIRInstructionClass<MoveToRegOp> TYPE = LIRInstructionClass.create(MoveToRegOp.class);
76 
77         @Def({REG, STACK, HINT}) protected AllocatableValue result;
78         @Use({REG, STACK}) protected AllocatableValue input;
79 
MoveToRegOp(AllocatableValue result, AllocatableValue input)80         public MoveToRegOp(AllocatableValue result, AllocatableValue input) {
81             super(TYPE);
82             this.result = result;
83             this.input = input;
84         }
85 
86         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)87         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
88             move(crb, masm, result, input);
89         }
90 
91         @Override
getInput()92         public AllocatableValue getInput() {
93             return input;
94         }
95 
96         @Override
getResult()97         public AllocatableValue getResult() {
98             return result;
99         }
100     }
101 
102     @Opcode("VMOVE")
103     public static final class MoveFromRegOp extends AMD64LIRInstruction implements ValueMoveOp {
104         public static final LIRInstructionClass<MoveFromRegOp> TYPE = LIRInstructionClass.create(MoveFromRegOp.class);
105 
106         @Def({REG, STACK}) protected AllocatableValue result;
107         @Use({REG, HINT}) protected AllocatableValue input;
108 
MoveFromRegOp(AllocatableValue result, AllocatableValue input)109         public MoveFromRegOp(AllocatableValue result, AllocatableValue input) {
110             super(TYPE);
111             this.result = result;
112             this.input = input;
113         }
114 
115         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)116         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
117             move(crb, masm, result, input);
118         }
119 
120         @Override
getInput()121         public AllocatableValue getInput() {
122             return input;
123         }
124 
125         @Override
getResult()126         public AllocatableValue getResult() {
127             return result;
128         }
129     }
130 
131     @Opcode("VMOVE")
132     public static class MoveFromConstOp extends AMD64LIRInstruction implements LoadConstantOp {
133         public static final LIRInstructionClass<MoveFromConstOp> TYPE = LIRInstructionClass.create(MoveFromConstOp.class);
134 
135         @Def({REG, STACK}) protected AllocatableValue result;
136         private final JavaConstant input;
137 
MoveFromConstOp(AllocatableValue result, JavaConstant input)138         public MoveFromConstOp(AllocatableValue result, JavaConstant input) {
139             super(TYPE);
140             this.result = result;
141             this.input = input;
142         }
143 
144         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)145         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
146             if (isRegister(result)) {
147                 const2reg(crb, masm, (RegisterValue) result, input);
148             } else {
149                 assert isStackSlot(result);
150                 AMD64Move.const2stack(crb, masm, result, input);
151             }
152         }
153 
154         @Override
getConstant()155         public Constant getConstant() {
156             return input;
157         }
158 
159         @Override
getResult()160         public AllocatableValue getResult() {
161             return result;
162         }
163     }
164 
165     @Opcode("VSTACKMOVE")
166     public static final class StackMoveOp extends AMD64LIRInstruction implements ValueMoveOp {
167         public static final LIRInstructionClass<StackMoveOp> TYPE = LIRInstructionClass.create(StackMoveOp.class);
168 
169         @Def({STACK}) protected AllocatableValue result;
170         @Use({STACK, HINT}) protected AllocatableValue input;
171         @Alive({STACK, UNINITIALIZED}) private AllocatableValue backupSlot;
172 
173         private Register scratch;
174 
StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot)175         public StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot) {
176             super(TYPE);
177             this.result = result;
178             this.input = input;
179             this.backupSlot = backupSlot;
180             this.scratch = scratch;
181         }
182 
183         @Override
getInput()184         public AllocatableValue getInput() {
185             return input;
186         }
187 
188         @Override
getResult()189         public AllocatableValue getResult() {
190             return result;
191         }
192 
193         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)194         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
195             // backup scratch register
196             move(crb, masm, backupSlot, scratch.asValue(backupSlot.getValueKind()));
197             // move stack slot
198             move(crb, masm, scratch.asValue(getInput().getValueKind()), getInput());
199             move(crb, masm, getResult(), scratch.asValue(getResult().getValueKind()));
200             // restore scratch register
201             move(crb, masm, scratch.asValue(backupSlot.getValueKind()), backupSlot);
202 
203         }
204     }
205 
206     public abstract static class VectorMemOp extends AMD64VectorInstruction {
207 
208         protected final VexMoveOp op;
209 
210         @Use({COMPOSITE}) protected AMD64AddressValue address;
211         @State protected LIRFrameState state;
212 
VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state)213         protected VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state) {
214             super(c, size);
215             this.op = op;
216             this.address = address;
217             this.state = state;
218         }
219 
emitMemAccess(AMD64MacroAssembler masm)220         protected abstract void emitMemAccess(AMD64MacroAssembler masm);
221 
222         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)223         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
224             if (state != null) {
225                 crb.recordImplicitException(masm.position(), state);
226             }
227             emitMemAccess(masm);
228         }
229     }
230 
231     public static final class VectorLoadOp extends VectorMemOp {
232         public static final LIRInstructionClass<VectorLoadOp> TYPE = LIRInstructionClass.create(VectorLoadOp.class);
233 
234         @Def({REG}) protected AllocatableValue result;
235 
VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state)236         public VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state) {
237             super(TYPE, size, op, address, state);
238             this.result = result;
239         }
240 
241         @Override
emitMemAccess(AMD64MacroAssembler masm)242         public void emitMemAccess(AMD64MacroAssembler masm) {
243             op.emit(masm, size, asRegister(result), address.toAddress());
244         }
245     }
246 
247     public static class VectorStoreOp extends VectorMemOp {
248         public static final LIRInstructionClass<VectorStoreOp> TYPE = LIRInstructionClass.create(VectorStoreOp.class);
249 
250         @Use({REG}) protected AllocatableValue input;
251 
VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state)252         public VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state) {
253             super(TYPE, size, op, address, state);
254             this.input = input;
255         }
256 
257         @Override
emitMemAccess(AMD64MacroAssembler masm)258         public void emitMemAccess(AMD64MacroAssembler masm) {
259             op.emit(masm, size, address.toAddress(), asRegister(input));
260         }
261     }
262 
263     @Opcode("SAVE_REGISTER")
264     public static class SaveRegistersOp extends AMD64SaveRegistersOp {
265         public static final LIRInstructionClass<SaveRegistersOp> TYPE = LIRInstructionClass.create(SaveRegistersOp.class);
266 
SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots)267         public SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots) {
268             super(TYPE, savedRegisters, slots);
269         }
270 
271         @Override
saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register)272         protected void saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register) {
273             AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
274             if (kind.isXMM()) {
275                 VexMoveOp op;
276                 if (kind.getVectorLength() > 1) {
277                     op = getVectorMoveOp(kind.getScalar());
278                 } else {
279                     op = getScalarMoveOp(kind);
280                 }
281 
282                 AMD64Address addr = (AMD64Address) crb.asAddress(result);
283                 op.emit(masm, AVXKind.getRegisterSize(kind), addr, register);
284             } else {
285                 super.saveRegister(crb, masm, result, register);
286             }
287         }
288     }
289 
290     @Opcode("RESTORE_REGISTER")
291     public static final class RestoreRegistersOp extends AMD64RestoreRegistersOp {
292         public static final LIRInstructionClass<RestoreRegistersOp> TYPE = LIRInstructionClass.create(RestoreRegistersOp.class);
293 
RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save)294         public RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save) {
295             super(TYPE, source, save);
296         }
297 
298         @Override
restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input)299         protected void restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input) {
300             AMD64Kind kind = (AMD64Kind) input.getPlatformKind();
301             if (kind.isXMM()) {
302                 VexMoveOp op;
303                 if (kind.getVectorLength() > 1) {
304                     op = getVectorMoveOp(kind.getScalar());
305                 } else {
306                     op = getScalarMoveOp(kind);
307                 }
308 
309                 AMD64Address addr = (AMD64Address) crb.asAddress(input);
310                 op.emit(masm, AVXKind.getRegisterSize(kind), register, addr);
311             } else {
312                 super.restoreRegister(crb, masm, register, input);
313             }
314         }
315     }
316 
getScalarMoveOp(AMD64Kind kind)317     private static VexMoveOp getScalarMoveOp(AMD64Kind kind) {
318         switch (kind) {
319             case SINGLE:
320                 return VMOVSS;
321             case DOUBLE:
322                 return VMOVSD;
323             default:
324                 throw GraalError.shouldNotReachHere();
325         }
326     }
327 
getVectorMoveOp(AMD64Kind kind)328     private static VexMoveOp getVectorMoveOp(AMD64Kind kind) {
329         switch (kind) {
330             case SINGLE:
331                 return VMOVUPS;
332             case DOUBLE:
333                 return VMOVUPD;
334             default:
335                 return VMOVDQU32;
336         }
337     }
338 
getVectorMemMoveOp(AMD64Kind kind)339     private static VexMoveOp getVectorMemMoveOp(AMD64Kind kind) {
340         switch (AVXKind.getDataSize(kind)) {
341             case DWORD:
342                 return VMOVD;
343             case QWORD:
344                 return VMOVQ;
345             default:
346                 return getVectorMoveOp(kind.getScalar());
347         }
348     }
349 
move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input)350     private static void move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input) {
351         VexMoveOp op;
352         AVXSize size;
353         AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
354         if (kind.getVectorLength() > 1) {
355             size = AVXKind.getRegisterSize(kind);
356             if (isRegister(input) && isRegister(result)) {
357                 op = getVectorMoveOp(kind.getScalar());
358             } else {
359                 op = getVectorMemMoveOp(kind);
360             }
361         } else {
362             size = AVXSize.XMM;
363             if (isRegister(input) && isRegister(result)) {
364                 op = getVectorMoveOp(kind);
365             } else {
366                 op = getScalarMoveOp(kind);
367             }
368         }
369 
370         if (isRegister(input)) {
371             if (isRegister(result)) {
372                 if (!asRegister(input).equals(asRegister(result))) {
373                     op.emit(masm, size, asRegister(result), asRegister(input));
374                 }
375             } else {
376                 assert isStackSlot(result);
377                 op.emit(masm, size, (AMD64Address) crb.asAddress(result), asRegister(input));
378             }
379         } else {
380             assert isStackSlot(input) && isRegister(result);
381             op.emit(masm, size, asRegister(result), (AMD64Address) crb.asAddress(input));
382         }
383     }
384 
const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input)385     private static void const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input) {
386         if (input.isDefaultForKind()) {
387             AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
388             Register register = result.getRegister();
389             VXORPD.emit(masm, AVXKind.getRegisterSize(kind), register, register, register);
390             return;
391         }
392 
393         AMD64Address address;
394         switch (input.getJavaKind()) {
395             case Float:
396                 address = (AMD64Address) crb.asFloatConstRef(input);
397                 break;
398 
399             case Double:
400                 address = (AMD64Address) crb.asDoubleConstRef(input);
401                 break;
402 
403             default:
404                 throw GraalError.shouldNotReachHere();
405         }
406         VexMoveOp op = getScalarMoveOp((AMD64Kind) result.getPlatformKind());
407         op.emit(masm, AVXSize.XMM, asRegister(result), address);
408     }
409 
410     public static final class AVXMoveToIntOp extends AMD64LIRInstruction {
411         public static final LIRInstructionClass<AVXMoveToIntOp> TYPE = LIRInstructionClass.create(AVXMoveToIntOp.class);
412 
413         @Opcode private final VexMoveOp opcode;
414 
415         @Def({REG, STACK}) protected AllocatableValue result;
416         @Use({REG}) protected AllocatableValue input;
417 
AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input)418         public AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input) {
419             super(TYPE);
420             this.opcode = opcode;
421             this.result = result;
422             this.input = input;
423         }
424 
425         @Override
emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm)426         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
427             if (isRegister(result)) {
428                 opcode.emitReverse(masm, AVXSize.XMM, asRegister(result), asRegister(input));
429             } else {
430                 opcode.emit(masm, AVXSize.XMM, (AMD64Address) crb.asAddress(result), asRegister(input));
431             }
432         }
433     }
434 }
435