1 /*
2  * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.
8  *
9  * This code is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * version 2 for more details (a copy is included in the LICENSE file that
13  * accompanied this code).
14  *
15  * You should have received a copy of the GNU General Public License version
16  * 2 along with this work; if not, write to the Free Software Foundation,
17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20  * or visit www.oracle.com if you need additional information or have any
21  * questions.
22  */
23 
24 
25 package org.graalvm.compiler.replacements;
26 
27 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_READ;
28 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_WRITE;
29 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_READ;
30 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_WRITE;
31 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile;
32 import static jdk.vm.ci.meta.DeoptimizationReason.BoundsCheckException;
33 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException;
34 import static org.graalvm.compiler.nodes.NamedLocationIdentity.ARRAY_LENGTH_LOCATION;
35 import static org.graalvm.compiler.nodes.java.ArrayLengthNode.readArrayLength;
36 import static org.graalvm.compiler.nodes.util.GraphUtil.skipPiWhileNonNull;
37 
38 import java.nio.ByteOrder;
39 import java.util.ArrayList;
40 import java.util.BitSet;
41 import java.util.List;
42 
43 import org.graalvm.compiler.api.directives.GraalDirectives;
44 import org.graalvm.compiler.api.replacements.Snippet;
45 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
46 import org.graalvm.compiler.core.common.LIRKind;
47 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
48 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
49 import org.graalvm.compiler.core.common.type.IntegerStamp;
50 import org.graalvm.compiler.core.common.type.ObjectStamp;
51 import org.graalvm.compiler.core.common.type.Stamp;
52 import org.graalvm.compiler.core.common.type.StampFactory;
53 import org.graalvm.compiler.core.common.type.TypeReference;
54 import org.graalvm.compiler.debug.DebugCloseable;
55 import org.graalvm.compiler.debug.DebugHandlersFactory;
56 import org.graalvm.compiler.debug.GraalError;
57 import org.graalvm.compiler.graph.Node;
58 import org.graalvm.compiler.nodeinfo.InputType;
59 import org.graalvm.compiler.nodes.CompressionNode.CompressionOp;
60 import org.graalvm.compiler.nodes.ConstantNode;
61 import org.graalvm.compiler.nodes.FieldLocationIdentity;
62 import org.graalvm.compiler.nodes.FixedNode;
63 import org.graalvm.compiler.nodes.LogicNode;
64 import org.graalvm.compiler.nodes.NamedLocationIdentity;
65 import org.graalvm.compiler.nodes.NodeView;
66 import org.graalvm.compiler.nodes.PiNode;
67 import org.graalvm.compiler.nodes.StructuredGraph;
68 import org.graalvm.compiler.nodes.ValueNode;
69 import org.graalvm.compiler.nodes.calc.AddNode;
70 import org.graalvm.compiler.nodes.calc.ConditionalNode;
71 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
72 import org.graalvm.compiler.nodes.calc.IntegerConvertNode;
73 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
74 import org.graalvm.compiler.nodes.calc.IsNullNode;
75 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
76 import org.graalvm.compiler.nodes.calc.NarrowNode;
77 import org.graalvm.compiler.nodes.calc.RightShiftNode;
78 import org.graalvm.compiler.nodes.calc.SignExtendNode;
79 import org.graalvm.compiler.nodes.calc.SubNode;
80 import org.graalvm.compiler.nodes.calc.UnpackEndianHalfNode;
81 import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
82 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
83 import org.graalvm.compiler.nodes.extended.BoxNode;
84 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
85 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
86 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
87 import org.graalvm.compiler.nodes.extended.GuardingNode;
88 import org.graalvm.compiler.nodes.extended.JavaReadNode;
89 import org.graalvm.compiler.nodes.extended.JavaWriteNode;
90 import org.graalvm.compiler.nodes.extended.LoadArrayComponentHubNode;
91 import org.graalvm.compiler.nodes.extended.LoadHubNode;
92 import org.graalvm.compiler.nodes.extended.MembarNode;
93 import org.graalvm.compiler.nodes.extended.RawLoadNode;
94 import org.graalvm.compiler.nodes.extended.RawStoreNode;
95 import org.graalvm.compiler.nodes.extended.UnboxNode;
96 import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode;
97 import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode;
98 import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
99 import org.graalvm.compiler.nodes.java.AccessIndexedNode;
100 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
101 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode;
102 import org.graalvm.compiler.nodes.java.FinalFieldBarrierNode;
103 import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode;
104 import org.graalvm.compiler.nodes.java.InstanceOfNode;
105 import org.graalvm.compiler.nodes.java.LoadFieldNode;
106 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
107 import org.graalvm.compiler.nodes.java.LogicCompareAndSwapNode;
108 import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode;
109 import org.graalvm.compiler.nodes.java.MonitorEnterNode;
110 import org.graalvm.compiler.nodes.java.MonitorIdNode;
111 import org.graalvm.compiler.nodes.java.NewArrayNode;
112 import org.graalvm.compiler.nodes.java.NewInstanceNode;
113 import org.graalvm.compiler.nodes.java.RawMonitorEnterNode;
114 import org.graalvm.compiler.nodes.java.StoreFieldNode;
115 import org.graalvm.compiler.nodes.java.StoreIndexedNode;
116 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode;
117 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode;
118 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode;
119 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
120 import org.graalvm.compiler.nodes.memory.ReadNode;
121 import org.graalvm.compiler.nodes.memory.WriteNode;
122 import org.graalvm.compiler.nodes.memory.address.AddressNode;
123 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
124 import org.graalvm.compiler.nodes.spi.Lowerable;
125 import org.graalvm.compiler.nodes.spi.LoweringProvider;
126 import org.graalvm.compiler.nodes.spi.LoweringTool;
127 import org.graalvm.compiler.nodes.type.StampTool;
128 import org.graalvm.compiler.nodes.util.GraphUtil;
129 import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode;
130 import org.graalvm.compiler.nodes.virtual.CommitAllocationNode;
131 import org.graalvm.compiler.nodes.virtual.VirtualArrayNode;
132 import org.graalvm.compiler.nodes.virtual.VirtualInstanceNode;
133 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
134 import org.graalvm.compiler.options.OptionValues;
135 import org.graalvm.compiler.phases.util.Providers;
136 import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering;
137 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode;
138 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation;
139 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode;
140 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation;
141 import jdk.internal.vm.compiler.word.LocationIdentity;
142 
143 import jdk.vm.ci.code.CodeUtil;
144 import jdk.vm.ci.code.MemoryBarriers;
145 import jdk.vm.ci.code.TargetDescription;
146 import jdk.vm.ci.meta.DeoptimizationAction;
147 import jdk.vm.ci.meta.DeoptimizationReason;
148 import jdk.vm.ci.meta.JavaKind;
149 import jdk.vm.ci.meta.MetaAccessProvider;
150 import jdk.vm.ci.meta.ResolvedJavaField;
151 import jdk.vm.ci.meta.ResolvedJavaMethod;
152 import jdk.vm.ci.meta.ResolvedJavaType;
153 import jdk.vm.ci.meta.SpeculationLog;
154 
155 /**
156  * VM-independent lowerings for standard Java nodes. VM-specific methods are abstract and must be
157  * implemented by VM-specific subclasses.
158  */
159 public abstract class DefaultJavaLoweringProvider implements LoweringProvider {
160 
161     protected final MetaAccessProvider metaAccess;
162     protected final ForeignCallsProvider foreignCalls;
163     protected final TargetDescription target;
164     private final boolean useCompressedOops;
165     private final ResolvedJavaType objectArrayType;
166 
167     private BoxingSnippets.Templates boxingSnippets;
168     private ConstantStringIndexOfSnippets.Templates indexOfSnippets;
169 
DefaultJavaLoweringProvider(MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, TargetDescription target, boolean useCompressedOops)170     public DefaultJavaLoweringProvider(MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, TargetDescription target, boolean useCompressedOops) {
171         this.metaAccess = metaAccess;
172         this.foreignCalls = foreignCalls;
173         this.target = target;
174         this.useCompressedOops = useCompressedOops;
175         this.objectArrayType = metaAccess.lookupJavaType(Object[].class);
176     }
177 
initialize(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, Providers providers, SnippetReflectionProvider snippetReflection)178     public void initialize(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, Providers providers, SnippetReflectionProvider snippetReflection) {
179         boxingSnippets = new BoxingSnippets.Templates(options, factories, factory, providers, snippetReflection, target);
180         indexOfSnippets = new ConstantStringIndexOfSnippets.Templates(options, factories, providers, snippetReflection, target);
181         providers.getReplacements().registerSnippetTemplateCache(new SnippetCounterNode.SnippetCounterSnippets.Templates(options, factories, providers, snippetReflection, target));
182     }
183 
getTarget()184     public final TargetDescription getTarget() {
185         return target;
186     }
187 
188     @Override
189     @SuppressWarnings("try")
lower(Node n, LoweringTool tool)190     public void lower(Node n, LoweringTool tool) {
191         assert n instanceof Lowerable;
192         StructuredGraph graph = (StructuredGraph) n.graph();
193         try (DebugCloseable context = n.withNodeSourcePosition()) {
194             if (n instanceof LoadFieldNode) {
195                 lowerLoadFieldNode((LoadFieldNode) n, tool);
196             } else if (n instanceof StoreFieldNode) {
197                 lowerStoreFieldNode((StoreFieldNode) n, tool);
198             } else if (n instanceof LoadIndexedNode) {
199                 lowerLoadIndexedNode((LoadIndexedNode) n, tool);
200             } else if (n instanceof StoreIndexedNode) {
201                 lowerStoreIndexedNode((StoreIndexedNode) n, tool);
202             } else if (n instanceof ArrayLengthNode) {
203                 lowerArrayLengthNode((ArrayLengthNode) n, tool);
204             } else if (n instanceof LoadHubNode) {
205                 lowerLoadHubNode((LoadHubNode) n, tool);
206             } else if (n instanceof LoadArrayComponentHubNode) {
207                 lowerLoadArrayComponentHubNode((LoadArrayComponentHubNode) n);
208             } else if (n instanceof MonitorEnterNode) {
209                 lowerMonitorEnterNode((MonitorEnterNode) n, tool, graph);
210             } else if (n instanceof UnsafeCompareAndSwapNode) {
211                 lowerCompareAndSwapNode((UnsafeCompareAndSwapNode) n);
212             } else if (n instanceof UnsafeCompareAndExchangeNode) {
213                 lowerCompareAndExchangeNode((UnsafeCompareAndExchangeNode) n);
214             } else if (n instanceof AtomicReadAndWriteNode) {
215                 lowerAtomicReadAndWriteNode((AtomicReadAndWriteNode) n);
216             } else if (n instanceof RawLoadNode) {
217                 lowerUnsafeLoadNode((RawLoadNode) n, tool);
218             } else if (n instanceof UnsafeMemoryLoadNode) {
219                 lowerUnsafeMemoryLoadNode((UnsafeMemoryLoadNode) n);
220             } else if (n instanceof RawStoreNode) {
221                 lowerUnsafeStoreNode((RawStoreNode) n);
222             } else if (n instanceof UnsafeMemoryStoreNode) {
223                 lowerUnsafeMemoryStoreNode((UnsafeMemoryStoreNode) n);
224             } else if (n instanceof JavaReadNode) {
225                 lowerJavaReadNode((JavaReadNode) n);
226             } else if (n instanceof JavaWriteNode) {
227                 lowerJavaWriteNode((JavaWriteNode) n);
228             } else if (n instanceof CommitAllocationNode) {
229                 lowerCommitAllocationNode((CommitAllocationNode) n, tool);
230             } else if (n instanceof BoxNode) {
231                 boxingSnippets.lower((BoxNode) n, tool);
232             } else if (n instanceof UnboxNode) {
233                 boxingSnippets.lower((UnboxNode) n, tool);
234             } else if (n instanceof VerifyHeapNode) {
235                 lowerVerifyHeap((VerifyHeapNode) n);
236             } else if (n instanceof UnaryMathIntrinsicNode) {
237                 lowerUnaryMath((UnaryMathIntrinsicNode) n, tool);
238             } else if (n instanceof BinaryMathIntrinsicNode) {
239                 lowerBinaryMath((BinaryMathIntrinsicNode) n, tool);
240             } else if (n instanceof StringIndexOfNode) {
241                 lowerIndexOf((StringIndexOfNode) n);
242             } else if (n instanceof UnpackEndianHalfNode) {
243                 lowerSecondHalf((UnpackEndianHalfNode) n);
244             } else {
245                 throw GraalError.shouldNotReachHere("Node implementing Lowerable not handled: " + n);
246             }
247         }
248     }
249 
lowerSecondHalf(UnpackEndianHalfNode n)250     private void lowerSecondHalf(UnpackEndianHalfNode n) {
251         ByteOrder byteOrder = target.arch.getByteOrder();
252         n.lower(byteOrder);
253     }
254 
lowerIndexOf(StringIndexOfNode n)255     private void lowerIndexOf(StringIndexOfNode n) {
256         if (n.getArgument(3).isConstant()) {
257             SnippetLowering lowering = new SnippetLowering() {
258                 @Override
259                 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) {
260                     if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
261                         return;
262                     }
263                     indexOfSnippets.lower(node, tool);
264                 }
265             };
266             SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Char), n.stamp(NodeView.DEFAULT), n.toArgumentArray());
267             n.graph().add(snippetLower);
268             n.graph().replaceFixedWithFixed(n, snippetLower);
269         }
270     }
271 
lowerBinaryMath(BinaryMathIntrinsicNode math, LoweringTool tool)272     private void lowerBinaryMath(BinaryMathIntrinsicNode math, LoweringTool tool) {
273         if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
274             return;
275         }
276         ResolvedJavaMethod method = math.graph().method();
277         if (method != null) {
278             if (method.getAnnotation(Snippet.class) != null) {
279                 /*
280                  * In the context of the snippet use the LIR lowering instead of the Node lowering.
281                  */
282                 return;
283             }
284             if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) {
285                 /*
286                  * A root compilation of the intrinsic method should emit the full assembly
287                  * implementation.
288                  */
289                 return;
290             }
291 
292         }
293         ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation());
294         if (foreignCall != null) {
295             StructuredGraph graph = math.graph();
296             ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, toForeignCall(math.getOperation()), math.getX(), math.getY()));
297             graph.addAfterFixed(tool.lastFixedNode(), call);
298             math.replaceAtUsages(call);
299         }
300     }
301 
lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool)302     private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) {
303         if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
304             return;
305         }
306         ResolvedJavaMethod method = math.graph().method();
307         if (method != null) {
308             if (method.getAnnotation(Snippet.class) != null) {
309                 /*
310                  * In the context of the snippet use the LIR lowering instead of the Node lowering.
311                  */
312                 return;
313             }
314             if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) {
315                 /*
316                  * A root compilation of the intrinsic method should emit the full assembly
317                  * implementation.
318                  */
319                 return;
320             }
321 
322         }
323         ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation());
324         if (foreignCall != null) {
325             StructuredGraph graph = math.graph();
326             ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, foreignCall, math.getValue()));
327             graph.addAfterFixed(tool.lastFixedNode(), call);
328             math.replaceAtUsages(call);
329         }
330     }
331 
toForeignCall(UnaryOperation operation)332     protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) {
333         return operation.foreignCallDescriptor;
334     }
335 
toForeignCall(BinaryOperation operation)336     protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) {
337         return operation.foreignCallDescriptor;
338     }
339 
lowerVerifyHeap(VerifyHeapNode n)340     protected void lowerVerifyHeap(VerifyHeapNode n) {
341         GraphUtil.removeFixedWithUnusedInputs(n);
342     }
343 
createOffsetAddress(StructuredGraph graph, ValueNode object, long offset)344     protected AddressNode createOffsetAddress(StructuredGraph graph, ValueNode object, long offset) {
345         ValueNode o = ConstantNode.forIntegerKind(target.wordJavaKind, offset, graph);
346         return graph.unique(new OffsetAddressNode(object, o));
347     }
348 
createFieldAddress(StructuredGraph graph, ValueNode object, ResolvedJavaField field)349     protected AddressNode createFieldAddress(StructuredGraph graph, ValueNode object, ResolvedJavaField field) {
350         int offset = fieldOffset(field);
351         if (offset >= 0) {
352             return createOffsetAddress(graph, object, offset);
353         } else {
354             return null;
355         }
356     }
357 
getStorageKind(ResolvedJavaField field)358     protected abstract JavaKind getStorageKind(ResolvedJavaField field);
359 
lowerLoadFieldNode(LoadFieldNode loadField, LoweringTool tool)360     protected void lowerLoadFieldNode(LoadFieldNode loadField, LoweringTool tool) {
361         assert loadField.getStackKind() != JavaKind.Illegal;
362         StructuredGraph graph = loadField.graph();
363         ResolvedJavaField field = loadField.field();
364         ValueNode object = loadField.isStatic() ? staticFieldBase(graph, field) : loadField.object();
365         object = createNullCheckedValue(object, loadField, tool);
366         Stamp loadStamp = loadStamp(loadField.stamp(NodeView.DEFAULT), getStorageKind(field));
367 
368         AddressNode address = createFieldAddress(graph, object, field);
369         assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName();
370 
371         ReadNode memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field)));
372         ValueNode readValue = implicitLoadConvert(graph, getStorageKind(field), memoryRead);
373         loadField.replaceAtUsages(readValue);
374         graph.replaceFixed(loadField, memoryRead);
375 
376         if (loadField.isVolatile()) {
377             MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ));
378             graph.addBeforeFixed(memoryRead, preMembar);
379             MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ));
380             graph.addAfterFixed(memoryRead, postMembar);
381         }
382     }
383 
lowerStoreFieldNode(StoreFieldNode storeField, LoweringTool tool)384     protected void lowerStoreFieldNode(StoreFieldNode storeField, LoweringTool tool) {
385         StructuredGraph graph = storeField.graph();
386         ResolvedJavaField field = storeField.field();
387         ValueNode object = storeField.isStatic() ? staticFieldBase(graph, field) : storeField.object();
388         object = createNullCheckedValue(object, storeField, tool);
389         ValueNode value = implicitStoreConvert(graph, getStorageKind(storeField.field()), storeField.value());
390         AddressNode address = createFieldAddress(graph, object, field);
391         assert address != null;
392 
393         WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field())));
394         memoryWrite.setStateAfter(storeField.stateAfter());
395         graph.replaceFixedWithFixed(storeField, memoryWrite);
396 
397         if (storeField.isVolatile()) {
398             MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE));
399             graph.addBeforeFixed(memoryWrite, preMembar);
400             MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE));
401             graph.addAfterFixed(memoryWrite, postMembar);
402         }
403     }
404 
405     /**
406      * Create a PiNode on the index proving that the index is positive. On some platforms this is
407      * important to allow the index to be used as an int in the address mode.
408      */
createArrayIndexAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index, GuardingNode boundsCheck)409     public AddressNode createArrayIndexAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index, GuardingNode boundsCheck) {
410         IntegerStamp indexStamp = StampFactory.forInteger(32, 0, Integer.MAX_VALUE - 1);
411         ValueNode positiveIndex = graph.maybeAddOrUnique(PiNode.create(index, indexStamp, boundsCheck != null ? boundsCheck.asNode() : null));
412         return createArrayAddress(graph, array, elementKind, positiveIndex);
413     }
414 
createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index)415     public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index) {
416         ValueNode wordIndex;
417         if (target.wordSize > 4) {
418             wordIndex = graph.unique(new SignExtendNode(index, target.wordSize * 8));
419         } else {
420             assert target.wordSize == 4 : "unsupported word size";
421             wordIndex = index;
422         }
423 
424         int shift = CodeUtil.log2(arrayScalingFactor(elementKind));
425         ValueNode scaledIndex = graph.unique(new LeftShiftNode(wordIndex, ConstantNode.forInt(shift, graph)));
426 
427         int base = arrayBaseOffset(elementKind);
428         ValueNode offset = graph.unique(new AddNode(scaledIndex, ConstantNode.forIntegerKind(target.wordJavaKind, base, graph)));
429 
430         return graph.unique(new OffsetAddressNode(array, offset));
431     }
432 
lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool)433     protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) {
434         StructuredGraph graph = loadIndexed.graph();
435         ValueNode array = loadIndexed.array();
436         array = createNullCheckedValue(array, loadIndexed, tool);
437         JavaKind elementKind = loadIndexed.elementKind();
438         Stamp loadStamp = loadStamp(loadIndexed.stamp(NodeView.DEFAULT), elementKind);
439 
440         GuardingNode boundsCheck = getBoundsCheck(loadIndexed, array, tool);
441         AddressNode address = createArrayIndexAddress(graph, array, elementKind, loadIndexed.index(), boundsCheck);
442         ReadNode memoryRead = graph.add(new ReadNode(address, NamedLocationIdentity.getArrayLocation(elementKind), loadStamp, BarrierType.NONE));
443         memoryRead.setGuard(boundsCheck);
444         ValueNode readValue = implicitLoadConvert(graph, elementKind, memoryRead);
445 
446         loadIndexed.replaceAtUsages(readValue);
447         graph.replaceFixed(loadIndexed, memoryRead);
448     }
449 
lowerStoreIndexedNode(StoreIndexedNode storeIndexed, LoweringTool tool)450     protected void lowerStoreIndexedNode(StoreIndexedNode storeIndexed, LoweringTool tool) {
451         StructuredGraph graph = storeIndexed.graph();
452 
453         ValueNode value = storeIndexed.value();
454         ValueNode array = storeIndexed.array();
455 
456         array = this.createNullCheckedValue(array, storeIndexed, tool);
457 
458         GuardingNode boundsCheck = getBoundsCheck(storeIndexed, array, tool);
459 
460         JavaKind elementKind = storeIndexed.elementKind();
461 
462         LogicNode condition = null;
463         if (storeIndexed.getStoreCheck() == null && elementKind == JavaKind.Object && !StampTool.isPointerAlwaysNull(value)) {
464             /* Array store check. */
465             TypeReference arrayType = StampTool.typeReferenceOrNull(array);
466             if (arrayType != null && arrayType.isExact()) {
467                 ResolvedJavaType elementType = arrayType.getType().getComponentType();
468                 if (!elementType.isJavaLangObject()) {
469                     TypeReference typeReference = TypeReference.createTrusted(storeIndexed.graph().getAssumptions(), elementType);
470                     LogicNode typeTest = graph.addOrUniqueWithInputs(InstanceOfNode.create(typeReference, value));
471                     condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY);
472                 }
473             } else {
474                 /*
475                  * The guard on the read hub should be the null check of the array that was
476                  * introduced earlier.
477                  */
478                 ValueNode arrayClass = createReadHub(graph, array, tool);
479                 ValueNode componentHub = createReadArrayComponentHub(graph, arrayClass, storeIndexed);
480                 LogicNode typeTest = graph.unique(InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), componentHub, value, false));
481                 condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY);
482             }
483         }
484 
485         AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck);
486         WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value),
487                         arrayStoreBarrierType(storeIndexed.elementKind())));
488         memoryWrite.setGuard(boundsCheck);
489         if (condition != null) {
490             tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile);
491         }
492         memoryWrite.setStateAfter(storeIndexed.stateAfter());
493         graph.replaceFixedWithFixed(storeIndexed, memoryWrite);
494     }
495 
lowerArrayLengthNode(ArrayLengthNode arrayLengthNode, LoweringTool tool)496     protected void lowerArrayLengthNode(ArrayLengthNode arrayLengthNode, LoweringTool tool) {
497         arrayLengthNode.replaceAtUsages(createReadArrayLength(arrayLengthNode.array(), arrayLengthNode, tool));
498         StructuredGraph graph = arrayLengthNode.graph();
499         graph.removeFixed(arrayLengthNode);
500     }
501 
502     /**
503      * Creates a read node that read the array length and is guarded by a null-check.
504      *
505      * The created node is placed before {@code before} in the CFG.
506      */
createReadArrayLength(ValueNode array, FixedNode before, LoweringTool tool)507     protected ReadNode createReadArrayLength(ValueNode array, FixedNode before, LoweringTool tool) {
508         StructuredGraph graph = array.graph();
509         ValueNode canonicalArray = this.createNullCheckedValue(skipPiWhileNonNull(array), before, tool);
510         AddressNode address = createOffsetAddress(graph, canonicalArray, arrayLengthOffset());
511         ReadNode readArrayLength = graph.add(new ReadNode(address, ARRAY_LENGTH_LOCATION, StampFactory.positiveInt(), BarrierType.NONE));
512         graph.addBeforeFixed(before, readArrayLength);
513         return readArrayLength;
514     }
515 
lowerLoadHubNode(LoadHubNode loadHub, LoweringTool tool)516     protected void lowerLoadHubNode(LoadHubNode loadHub, LoweringTool tool) {
517         StructuredGraph graph = loadHub.graph();
518         if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
519             return;
520         }
521         if (graph.getGuardsStage().allowsFloatingGuards()) {
522             return;
523         }
524         ValueNode hub = createReadHub(graph, loadHub.getValue(), tool);
525         loadHub.replaceAtUsagesAndDelete(hub);
526     }
527 
lowerLoadArrayComponentHubNode(LoadArrayComponentHubNode loadHub)528     protected void lowerLoadArrayComponentHubNode(LoadArrayComponentHubNode loadHub) {
529         StructuredGraph graph = loadHub.graph();
530         ValueNode hub = createReadArrayComponentHub(graph, loadHub.getValue(), loadHub);
531         graph.replaceFixed(loadHub, hub);
532     }
533 
lowerMonitorEnterNode(MonitorEnterNode monitorEnter, LoweringTool tool, StructuredGraph graph)534     protected void lowerMonitorEnterNode(MonitorEnterNode monitorEnter, LoweringTool tool, StructuredGraph graph) {
535         ValueNode object = createNullCheckedValue(monitorEnter.object(), monitorEnter, tool);
536         ValueNode hub = graph.addOrUnique(LoadHubNode.create(object, tool.getStampProvider(), tool.getMetaAccess(), tool.getConstantReflection()));
537         RawMonitorEnterNode rawMonitorEnter = graph.add(new RawMonitorEnterNode(object, hub, monitorEnter.getMonitorId()));
538         rawMonitorEnter.setStateBefore(monitorEnter.stateBefore());
539         rawMonitorEnter.setStateAfter(monitorEnter.stateAfter());
540         graph.replaceFixedWithFixed(monitorEnter, rawMonitorEnter);
541     }
542 
lowerCompareAndSwapNode(UnsafeCompareAndSwapNode cas)543     protected void lowerCompareAndSwapNode(UnsafeCompareAndSwapNode cas) {
544         StructuredGraph graph = cas.graph();
545         JavaKind valueKind = cas.getValueKind();
546 
547         ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected());
548         ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue());
549 
550         AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset()));
551         BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue);
552         LogicCompareAndSwapNode atomicNode = graph.add(new LogicCompareAndSwapNode(address, cas.getLocationIdentity(), expectedValue, newValue, barrierType));
553         atomicNode.setStateAfter(cas.stateAfter());
554         graph.replaceFixedWithFixed(cas, atomicNode);
555     }
556 
lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas)557     protected void lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas) {
558         StructuredGraph graph = cas.graph();
559         JavaKind valueKind = cas.getValueKind();
560 
561         ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected());
562         ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue());
563 
564         AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset()));
565         BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue);
566         ValueCompareAndSwapNode atomicNode = graph.add(new ValueCompareAndSwapNode(address, expectedValue, newValue, cas.getLocationIdentity(), barrierType));
567         ValueNode coercedNode = implicitLoadConvert(graph, valueKind, atomicNode, true);
568         atomicNode.setStateAfter(cas.stateAfter());
569         cas.replaceAtUsages(coercedNode);
570         graph.replaceFixedWithFixed(cas, atomicNode);
571     }
572 
lowerAtomicReadAndWriteNode(AtomicReadAndWriteNode n)573     protected void lowerAtomicReadAndWriteNode(AtomicReadAndWriteNode n) {
574         StructuredGraph graph = n.graph();
575         JavaKind valueKind = n.getValueKind();
576 
577         ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue());
578 
579         AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset()));
580         BarrierType barrierType = guessStoreBarrierType(n.object(), n.newValue());
581         LIRKind lirAccessKind = LIRKind.fromJavaKind(target.arch, valueKind);
582         LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getLocationIdentity(), newValue, lirAccessKind, barrierType));
583         memoryRead.setStateAfter(n.stateAfter());
584 
585         ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead);
586         n.stateAfter().replaceFirstInput(n, memoryRead);
587         n.replaceAtUsages(readValue);
588         graph.replaceFixedWithFixed(n, memoryRead);
589     }
590 
591     /**
592      * @param tool utility for performing the lowering
593      */
lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool)594     protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) {
595         StructuredGraph graph = load.graph();
596         if (load instanceof GuardedUnsafeLoadNode) {
597             GuardedUnsafeLoadNode guardedLoad = (GuardedUnsafeLoadNode) load;
598             GuardingNode guard = guardedLoad.getGuard();
599             if (guard == null) {
600                 // can float freely if the guard folded away
601                 ReadNode memoryRead = createUnsafeRead(graph, load, null);
602                 memoryRead.setForceFixed(false);
603                 graph.replaceFixedWithFixed(load, memoryRead);
604             } else {
605                 // must be guarded, but flows below the guard
606                 ReadNode memoryRead = createUnsafeRead(graph, load, guard);
607                 graph.replaceFixedWithFixed(load, memoryRead);
608             }
609         } else {
610             // never had a guarding condition so it must be fixed, creation of the read will force
611             // it to be fixed
612             ReadNode memoryRead = createUnsafeRead(graph, load, null);
613             graph.replaceFixedWithFixed(load, memoryRead);
614         }
615     }
616 
createUnsafeAddress(StructuredGraph graph, ValueNode object, ValueNode offset)617     protected AddressNode createUnsafeAddress(StructuredGraph graph, ValueNode object, ValueNode offset) {
618         if (object.isConstant() && object.asConstant().isDefaultForKind()) {
619             return graph.addOrUniqueWithInputs(OffsetAddressNode.create(offset));
620         } else {
621             return graph.unique(new OffsetAddressNode(object, offset));
622         }
623     }
624 
createUnsafeRead(StructuredGraph graph, RawLoadNode load, GuardingNode guard)625     protected ReadNode createUnsafeRead(StructuredGraph graph, RawLoadNode load, GuardingNode guard) {
626         boolean compressible = load.accessKind() == JavaKind.Object;
627         JavaKind readKind = load.accessKind();
628         Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, compressible);
629         AddressNode address = createUnsafeAddress(graph, load.object(), load.offset());
630         ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE));
631         if (guard == null) {
632             // An unsafe read must not float otherwise it may float above
633             // a test guaranteeing the read is safe.
634             memoryRead.setForceFixed(true);
635         } else {
636             memoryRead.setGuard(guard);
637         }
638         ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, compressible), readKind);
639         load.replaceAtUsages(readValue);
640         return memoryRead;
641     }
642 
lowerUnsafeMemoryLoadNode(UnsafeMemoryLoadNode load)643     protected void lowerUnsafeMemoryLoadNode(UnsafeMemoryLoadNode load) {
644         StructuredGraph graph = load.graph();
645         JavaKind readKind = load.getKind();
646         assert readKind != JavaKind.Object;
647         Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, false);
648         AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(load.getAddress()));
649         ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE));
650         // An unsafe read must not float otherwise it may float above
651         // a test guaranteeing the read is safe.
652         memoryRead.setForceFixed(true);
653         ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, false), readKind);
654         load.replaceAtUsages(readValue);
655         graph.replaceFixedWithFixed(load, memoryRead);
656     }
657 
performBooleanCoercionIfNecessary(ValueNode readValue, JavaKind readKind)658     private static ValueNode performBooleanCoercionIfNecessary(ValueNode readValue, JavaKind readKind) {
659         if (readKind == JavaKind.Boolean) {
660             StructuredGraph graph = readValue.graph();
661             IntegerEqualsNode eq = graph.addOrUnique(new IntegerEqualsNode(readValue, ConstantNode.forInt(0, graph)));
662             return graph.addOrUnique(new ConditionalNode(eq, ConstantNode.forBoolean(false, graph), ConstantNode.forBoolean(true, graph)));
663         }
664         return readValue;
665     }
666 
lowerUnsafeStoreNode(RawStoreNode store)667     protected void lowerUnsafeStoreNode(RawStoreNode store) {
668         StructuredGraph graph = store.graph();
669         boolean compressible = store.value().getStackKind() == JavaKind.Object;
670         JavaKind valueKind = store.accessKind();
671         ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible);
672         AddressNode address = createUnsafeAddress(graph, store.object(), store.offset());
673         WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, unsafeStoreBarrierType(store)));
674         write.setStateAfter(store.stateAfter());
675         graph.replaceFixedWithFixed(store, write);
676     }
677 
lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store)678     protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) {
679         StructuredGraph graph = store.graph();
680         assert store.getValue().getStackKind() != JavaKind.Object;
681         JavaKind valueKind = store.getKind();
682         ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false);
683         AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(store.getAddress()));
684         WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, BarrierType.NONE));
685         write.setStateAfter(store.stateAfter());
686         graph.replaceFixedWithFixed(store, write);
687     }
688 
lowerJavaReadNode(JavaReadNode read)689     protected void lowerJavaReadNode(JavaReadNode read) {
690         StructuredGraph graph = read.graph();
691         JavaKind valueKind = read.getReadKind();
692         Stamp loadStamp = loadStamp(read.stamp(NodeView.DEFAULT), valueKind, read.isCompressible());
693 
694         ReadNode memoryRead = graph.add(new ReadNode(read.getAddress(), read.getLocationIdentity(), loadStamp, read.getBarrierType()));
695         GuardingNode guard = read.getGuard();
696         ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead, read.isCompressible());
697         if (guard == null) {
698             // An unsafe read must not float otherwise it may float above
699             // a test guaranteeing the read is safe.
700             memoryRead.setForceFixed(true);
701         } else {
702             memoryRead.setGuard(guard);
703         }
704         read.replaceAtUsages(readValue);
705         graph.replaceFixed(read, memoryRead);
706     }
707 
lowerJavaWriteNode(JavaWriteNode write)708     protected void lowerJavaWriteNode(JavaWriteNode write) {
709         StructuredGraph graph = write.graph();
710         ValueNode value = implicitStoreConvert(graph, write.getWriteKind(), write.value(), write.isCompressible());
711         WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getLocationIdentity(), value, write.getBarrierType()));
712         memoryWrite.setStateAfter(write.stateAfter());
713         graph.replaceFixedWithFixed(write, memoryWrite);
714         memoryWrite.setGuard(write.getGuard());
715     }
716 
717     @SuppressWarnings("try")
lowerCommitAllocationNode(CommitAllocationNode commit, LoweringTool tool)718     protected void lowerCommitAllocationNode(CommitAllocationNode commit, LoweringTool tool) {
719         StructuredGraph graph = commit.graph();
720         if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) {
721             List<AbstractNewObjectNode> recursiveLowerings = new ArrayList<>();
722 
723             ValueNode[] allocations = new ValueNode[commit.getVirtualObjects().size()];
724             BitSet omittedValues = new BitSet();
725             int valuePos = 0;
726             for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
727                 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex);
728                 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) {
729                     int entryCount = virtual.entryCount();
730                     AbstractNewObjectNode newObject;
731                     if (virtual instanceof VirtualInstanceNode) {
732                         newObject = graph.add(createNewInstanceFromVirtual(virtual));
733                     } else {
734                         newObject = graph.add(createNewArrayFromVirtual(virtual, ConstantNode.forInt(entryCount, graph)));
735                     }
736 
737                     recursiveLowerings.add(newObject);
738                     graph.addBeforeFixed(commit, newObject);
739                     allocations[objIndex] = newObject;
740                     for (int i = 0; i < entryCount; i++) {
741                         ValueNode value = commit.getValues().get(valuePos);
742                         if (value instanceof VirtualObjectNode) {
743                             value = allocations[commit.getVirtualObjects().indexOf(value)];
744                         }
745                         if (value == null) {
746                             omittedValues.set(valuePos);
747                         } else if (!(value.isConstant() && value.asConstant().isDefaultForKind())) {
748                             // Constant.illegal is always the defaultForKind, so it is skipped
749                             JavaKind valueKind = value.getStackKind();
750                             JavaKind entryKind = virtual.entryKind(i);
751 
752                             // Truffle requires some leniency in terms of what can be put where:
753                             assert valueKind.getStackKind() == entryKind.getStackKind() ||
754                                             (valueKind == JavaKind.Long || valueKind == JavaKind.Double || (valueKind == JavaKind.Int && virtual instanceof VirtualArrayNode));
755                             AddressNode address = null;
756                             BarrierType barrierType = null;
757                             if (virtual instanceof VirtualInstanceNode) {
758                                 ResolvedJavaField field = ((VirtualInstanceNode) virtual).field(i);
759                                 long offset = fieldOffset(field);
760                                 if (offset >= 0) {
761                                     address = createOffsetAddress(graph, newObject, offset);
762                                     barrierType = fieldInitializationBarrier(entryKind);
763                                 }
764                             } else {
765                                 address = createOffsetAddress(graph, newObject, tool.getMetaAccess().getArrayBaseOffset(entryKind) + i * arrayScalingFactor(entryKind));
766                                 barrierType = arrayInitializationBarrier(entryKind);
767                             }
768                             if (address != null) {
769                                 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType);
770                                 graph.addAfterFixed(newObject, graph.add(write));
771                             }
772                         }
773                         valuePos++;
774                     }
775                 }
776             }
777             valuePos = 0;
778 
779             for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
780                 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex);
781                 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) {
782                     int entryCount = virtual.entryCount();
783                     ValueNode newObject = allocations[objIndex];
784                     for (int i = 0; i < entryCount; i++) {
785                         if (omittedValues.get(valuePos)) {
786                             ValueNode value = commit.getValues().get(valuePos);
787                             assert value instanceof VirtualObjectNode;
788                             ValueNode allocValue = allocations[commit.getVirtualObjects().indexOf(value)];
789                             if (!(allocValue.isConstant() && allocValue.asConstant().isDefaultForKind())) {
790                                 assert virtual.entryKind(i) == JavaKind.Object && allocValue.getStackKind() == JavaKind.Object;
791                                 AddressNode address;
792                                 BarrierType barrierType;
793                                 if (virtual instanceof VirtualInstanceNode) {
794                                     VirtualInstanceNode virtualInstance = (VirtualInstanceNode) virtual;
795                                     address = createFieldAddress(graph, newObject, virtualInstance.field(i));
796                                     barrierType = BarrierType.IMPRECISE;
797                                 } else {
798                                     address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph));
799                                     barrierType = BarrierType.PRECISE;
800                                 }
801                                 if (address != null) {
802                                     WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType);
803                                     graph.addBeforeFixed(commit, graph.add(write));
804                                 }
805                             }
806                         }
807                         valuePos++;
808                     }
809                 }
810             }
811 
812             finishAllocatedObjects(tool, commit, allocations);
813             graph.removeFixed(commit);
814 
815             for (AbstractNewObjectNode recursiveLowering : recursiveLowerings) {
816                 recursiveLowering.lower(tool);
817             }
818         }
819 
820     }
821 
createNewInstanceFromVirtual(VirtualObjectNode virtual)822     public NewInstanceNode createNewInstanceFromVirtual(VirtualObjectNode virtual) {
823         return new NewInstanceNode(virtual.type(), true);
824     }
825 
createNewArrayFromVirtual(VirtualObjectNode virtual, ValueNode length)826     protected NewArrayNode createNewArrayFromVirtual(VirtualObjectNode virtual, ValueNode length) {
827         return new NewArrayNode(((VirtualArrayNode) virtual).componentType(), length, true);
828     }
829 
finishAllocatedObjects(LoweringTool tool, CommitAllocationNode commit, ValueNode[] allocations)830     public void finishAllocatedObjects(LoweringTool tool, CommitAllocationNode commit, ValueNode[] allocations) {
831         StructuredGraph graph = commit.graph();
832         for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
833             FixedValueAnchorNode anchor = graph.add(new FixedValueAnchorNode(allocations[objIndex]));
834             allocations[objIndex] = anchor;
835             graph.addBeforeFixed(commit, anchor);
836         }
837         /*
838          * Note that the FrameState that is assigned to these MonitorEnterNodes isn't the correct
839          * state. It will be the state from before the allocation occurred instead of a valid state
840          * after the locking is performed. In practice this should be fine since these are newly
841          * allocated objects. The bytecodes themselves permit allocating an object, doing a
842          * monitorenter and then dropping all references to the object which would produce the same
843          * state, though that would normally produce an IllegalMonitorStateException. In HotSpot
844          * some form of fast path locking should always occur so the FrameState should never
845          * actually be used.
846          */
847         ArrayList<MonitorEnterNode> enters = null;
848         for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
849             List<MonitorIdNode> locks = commit.getLocks(objIndex);
850             if (locks.size() > 1) {
851                 // Ensure that the lock operations are performed in lock depth order
852                 ArrayList<MonitorIdNode> newList = new ArrayList<>(locks);
853                 newList.sort((a, b) -> Integer.compare(a.getLockDepth(), b.getLockDepth()));
854                 locks = newList;
855             }
856             int lastDepth = -1;
857             for (MonitorIdNode monitorId : locks) {
858                 assert lastDepth < monitorId.getLockDepth();
859                 lastDepth = monitorId.getLockDepth();
860                 MonitorEnterNode enter = graph.add(new MonitorEnterNode(allocations[objIndex], monitorId));
861                 graph.addBeforeFixed(commit, enter);
862                 if (enters == null) {
863                     enters = new ArrayList<>();
864                 }
865                 enters.add(enter);
866             }
867         }
868         for (Node usage : commit.usages().snapshot()) {
869             if (usage instanceof AllocatedObjectNode) {
870                 AllocatedObjectNode addObject = (AllocatedObjectNode) usage;
871                 int index = commit.getVirtualObjects().indexOf(addObject.getVirtualObject());
872                 addObject.replaceAtUsagesAndDelete(allocations[index]);
873             } else {
874                 assert enters != null;
875                 commit.replaceAtUsages(InputType.Memory, enters.get(enters.size() - 1));
876             }
877         }
878         if (enters != null) {
879             for (MonitorEnterNode enter : enters) {
880                 enter.lower(tool);
881             }
882         }
883         assert commit.hasNoUsages();
884         insertAllocationBarrier(commit, graph);
885     }
886 
887     /**
888      * Insert the required {@link MemoryBarriers#STORE_STORE} barrier for an allocation and also
889      * include the {@link MemoryBarriers#LOAD_STORE} required for final fields if any final fields
890      * are being written, as if {@link FinalFieldBarrierNode} were emitted.
891      */
892     private static void insertAllocationBarrier(CommitAllocationNode commit, StructuredGraph graph) {
893         int barrier = MemoryBarriers.STORE_STORE;
894         outer: for (VirtualObjectNode vobj : commit.getVirtualObjects()) {
895             for (ResolvedJavaField field : vobj.type().getInstanceFields(true)) {
896                 if (field.isFinal()) {
897                     barrier = barrier | MemoryBarriers.LOAD_STORE;
898                     break outer;
899                 }
900             }
901         }
902         graph.addAfterFixed(commit, graph.add(new MembarNode(barrier, LocationIdentity.init())));
903     }
904 
905     /**
906      * @param field the field whose barrier type should be returned
907      */
908     protected BarrierType fieldLoadBarrierType(ResolvedJavaField field) {
909         return BarrierType.NONE;
910     }
911 
912     protected BarrierType fieldStoreBarrierType(ResolvedJavaField field) {
913         if (field.getJavaKind() == JavaKind.Object) {
914             return BarrierType.IMPRECISE;
915         }
916         return BarrierType.NONE;
917     }
918 
919     protected BarrierType arrayStoreBarrierType(JavaKind elementKind) {
920         if (elementKind == JavaKind.Object) {
921             return BarrierType.PRECISE;
922         }
923         return BarrierType.NONE;
924     }
925 
926     public BarrierType fieldInitializationBarrier(JavaKind entryKind) {
927         return entryKind == JavaKind.Object ? BarrierType.IMPRECISE : BarrierType.NONE;
928     }
929 
930     public BarrierType arrayInitializationBarrier(JavaKind entryKind) {
931         return entryKind == JavaKind.Object ? BarrierType.PRECISE : BarrierType.NONE;
932     }
933 
934     private BarrierType unsafeStoreBarrierType(RawStoreNode store) {
935         if (!store.needsBarrier()) {
936             return BarrierType.NONE;
937         }
938         return guessStoreBarrierType(store.object(), store.value());
939     }
940 
941     private BarrierType guessStoreBarrierType(ValueNode object, ValueNode value) {
942         if (value.getStackKind() == JavaKind.Object && object.getStackKind() == JavaKind.Object) {
943             ResolvedJavaType type = StampTool.typeOrNull(object);
944             // Array types must use a precise barrier, so if the type is unknown or is a supertype
945             // of Object[] then treat it as an array.
946             if (type == null || type.isArray() || type.isAssignableFrom(objectArrayType)) {
947                 return BarrierType.PRECISE;
948             } else {
949                 return BarrierType.IMPRECISE;
950             }
951         }
952         return BarrierType.NONE;
953     }
954 
955     public abstract int fieldOffset(ResolvedJavaField field);
956 
957     public FieldLocationIdentity fieldLocationIdentity(ResolvedJavaField field) {
958         return new FieldLocationIdentity(field);
959     }
960 
961     public abstract ValueNode staticFieldBase(StructuredGraph graph, ResolvedJavaField field);
962 
963     public abstract int arrayLengthOffset();
964 
965     @Override
966     public int arrayScalingFactor(JavaKind elementKind) {
967         return target.arch.getPlatformKind(elementKind).getSizeInBytes();
968     }
969 
970     public Stamp loadStamp(Stamp stamp, JavaKind kind) {
971         return loadStamp(stamp, kind, true);
972     }
973 
974     private boolean useCompressedOops(JavaKind kind, boolean compressible) {
975         return kind == JavaKind.Object && compressible && useCompressedOops;
976     }
977 
978     protected abstract Stamp loadCompressedStamp(ObjectStamp stamp);
979 
980     /**
981      * @param compressible whether the stamp should be compressible
982      */
983     protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) {
984         if (useCompressedOops(kind, compressible)) {
985             return loadCompressedStamp((ObjectStamp) stamp);
986         }
987 
988         switch (kind) {
989             case Boolean:
990             case Byte:
991                 return IntegerStamp.OPS.getNarrow().foldStamp(32, 8, stamp);
992             case Char:
993             case Short:
994                 return IntegerStamp.OPS.getNarrow().foldStamp(32, 16, stamp);
995         }
996         return stamp;
997     }
998 
999     public final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value) {
1000         return implicitLoadConvert(graph, kind, value, true);
1001     }
1002 
1003     public ValueNode implicitLoadConvert(JavaKind kind, ValueNode value) {
1004         return implicitLoadConvert(kind, value, true);
1005     }
1006 
1007     protected final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) {
1008         ValueNode ret = implicitLoadConvert(kind, value, compressible);
1009         if (!ret.isAlive()) {
1010             ret = graph.addOrUnique(ret);
1011         }
1012         return ret;
1013     }
1014 
1015     protected abstract ValueNode newCompressionNode(CompressionOp op, ValueNode value);
1016 
1017     /**
1018      * @param compressible whether the convert should be compressible
1019      */
1020     protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) {
1021         if (useCompressedOops(kind, compressible)) {
1022             return newCompressionNode(CompressionOp.Uncompress, value);
1023         }
1024 
1025         switch (kind) {
1026             case Byte:
1027             case Short:
1028                 return new SignExtendNode(value, 32);
1029             case Boolean:
1030             case Char:
1031                 return new ZeroExtendNode(value, 32);
1032         }
1033         return value;
1034     }
1035 
1036     public final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value) {
1037         return implicitStoreConvert(graph, kind, value, true);
1038     }
1039 
1040     public ValueNode implicitStoreConvert(JavaKind kind, ValueNode value) {
1041         return implicitStoreConvert(kind, value, true);
1042     }
1043 
1044     protected final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) {
1045         ValueNode ret = implicitStoreConvert(kind, value, compressible);
1046         if (!ret.isAlive()) {
1047             ret = graph.addOrUnique(ret);
1048         }
1049         return ret;
1050     }
1051 
1052     /**
1053      * @param compressible whether the covert should be compressible
1054      */
1055     protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) {
1056         if (useCompressedOops(kind, compressible)) {
1057             return newCompressionNode(CompressionOp.Compress, value);
1058         }
1059 
1060         switch (kind) {
1061             case Boolean:
1062             case Byte:
1063                 return new NarrowNode(value, 8);
1064             case Char:
1065             case Short:
1066                 return new NarrowNode(value, 16);
1067         }
1068         return value;
1069     }
1070 
1071     protected abstract ValueNode createReadHub(StructuredGraph graph, ValueNode object, LoweringTool tool);
1072 
1073     protected abstract ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor);
1074 
1075     protected GuardingNode getBoundsCheck(AccessIndexedNode n, ValueNode array, LoweringTool tool) {
1076         if (n.getBoundsCheck() != null) {
1077             return n.getBoundsCheck();
1078         }
1079 
1080         StructuredGraph graph = n.graph();
1081         ValueNode arrayLength = readArrayLength(array, tool.getConstantReflection());
1082         if (arrayLength == null) {
1083             arrayLength = createReadArrayLength(array, n, tool);
1084         } else {
1085             arrayLength = arrayLength.isAlive() ? arrayLength : graph.addOrUniqueWithInputs(arrayLength);
1086         }
1087 
1088         LogicNode boundsCheck = IntegerBelowNode.create(n.index(), arrayLength, NodeView.DEFAULT);
1089         if (boundsCheck.isTautology()) {
1090             return null;
1091         }
1092         return tool.createGuard(n, graph.addOrUniqueWithInputs(boundsCheck), BoundsCheckException, InvalidateReprofile);
1093     }
1094 
1095     protected GuardingNode createNullCheck(ValueNode object, FixedNode before, LoweringTool tool) {
1096         if (StampTool.isPointerNonNull(object)) {
1097             return null;
1098         }
1099         return tool.createGuard(before, before.graph().unique(IsNullNode.create(object)), NullCheckException, InvalidateReprofile, SpeculationLog.NO_SPECULATION, true, null);
1100     }
1101 
1102     protected ValueNode createNullCheckedValue(ValueNode object, FixedNode before, LoweringTool tool) {
1103         GuardingNode nullCheck = createNullCheck(object, before, tool);
1104         if (nullCheck == null) {
1105             return object;
1106         }
1107         return before.graph().maybeAddOrUnique(PiNode.create(object, (object.stamp(NodeView.DEFAULT)).join(StampFactory.objectNonNull()), (ValueNode) nullCheck));
1108     }
1109 
1110     @Override
1111     public ValueNode reconstructArrayIndex(JavaKind elementKind, AddressNode address) {
1112         StructuredGraph graph = address.graph();
1113         ValueNode offset = ((OffsetAddressNode) address).getOffset();
1114 
1115         int base = arrayBaseOffset(elementKind);
1116         ValueNode scaledIndex = graph.unique(new SubNode(offset, ConstantNode.forIntegerStamp(offset.stamp(NodeView.DEFAULT), base, graph)));
1117 
1118         int shift = CodeUtil.log2(arrayScalingFactor(elementKind));
1119         ValueNode ret = graph.unique(new RightShiftNode(scaledIndex, ConstantNode.forInt(shift, graph)));
1120         return IntegerConvertNode.convert(ret, StampFactory.forKind(JavaKind.Int), graph, NodeView.DEFAULT);
1121     }
1122 }
1123