1 /*
2  * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.
8  *
9  * This code is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * version 2 for more details (a copy is included in the LICENSE file that
13  * accompanied this code).
14  *
15  * You should have received a copy of the GNU General Public License version
16  * 2 along with this work; if not, write to the Free Software Foundation,
17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20  * or visit www.oracle.com if you need additional information or have any
21  * questions.
22  */
23 
24 
25 package org.graalvm.compiler.hotspot.stubs;
26 
27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getArrayBaseOffset;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useFastTLABRefill;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
65 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
66 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
67 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
68 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
71 
72 import org.graalvm.compiler.api.replacements.Fold;
73 import org.graalvm.compiler.api.replacements.Snippet;
74 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
76 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
77 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
78 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
79 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
80 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
81 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
82 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
83 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
84 import org.graalvm.compiler.hotspot.word.KlassPointer;
85 import org.graalvm.compiler.nodes.ConstantNode;
86 import org.graalvm.compiler.options.OptionValues;
87 import org.graalvm.compiler.word.Word;
88 import jdk.internal.vm.compiler.word.WordFactory;
89 
90 import jdk.vm.ci.code.Register;
91 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
92 import jdk.vm.ci.meta.JavaKind;
93 
94 /**
95  * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
96  * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
97  * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
98  * runtime for to complete the allocation.
99  */
100 public class NewInstanceStub extends SnippetStub {
101 
NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage)102     public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
103         super("newInstance", options, providers, linkage);
104     }
105 
106     @Override
makeConstArgs()107     protected Object[] makeConstArgs() {
108         HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
109         int count = method.getSignature().getParameterCount(false);
110         Object[] args = new Object[count];
111         assert checkConstArg(1, "intArrayHub");
112         assert checkConstArg(2, "threadRegister");
113         assert checkConstArg(3, "options");
114         args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
115         args[2] = providers.getRegisters().getThreadRegister();
116         args[3] = options;
117         return args;
118     }
119 
allocate(Word thread, int size)120     private static Word allocate(Word thread, int size) {
121         Word top = readTlabTop(thread);
122         Word end = readTlabEnd(thread);
123         Word newTop = top.add(size);
124         /*
125          * this check might lead to problems if the TLAB is within 16GB of the address space end
126          * (checked in c++ code)
127          */
128         if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
129             writeTlabTop(thread, newTop);
130             return top;
131         }
132         return WordFactory.zero();
133     }
134 
135     @Fold
logging(OptionValues options)136     static boolean logging(OptionValues options) {
137         return StubOptions.TraceNewInstanceStub.getValue(options);
138     }
139 
140     /**
141      * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
142      * -XX:-UseTLAB).
143      *
144      * @param hub the hub of the object to be allocated
145      * @param intArrayHub the hub for {@code int[].class}
146      */
147     @Snippet
newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options)148     private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
149         /*
150          * The type is known to be an instance so Klass::_layout_helper is the instance size as a
151          * raw number
152          */
153         Word thread = registerAsWord(threadRegister);
154         boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
155         if (useFastTLABRefill(INJECTED_VMCONFIG) && !forceSlowPath(options) && inlineContiguousAllocationSupported && !useCMSIncrementalMode(INJECTED_VMCONFIG)) {
156             if (isInstanceKlassFullyInitialized(hub)) {
157                 int sizeInBytes = readLayoutHelper(hub);
158                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
159                 if (memory.notEqual(0)) {
160                     Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
161                     NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
162                     return verifyObject(memory.toObject());
163                 }
164             }
165         }
166 
167         if (logging(options)) {
168             printf("newInstance: calling new_instance_c\n");
169         }
170 
171         newInstanceC(NEW_INSTANCE_C, thread, hub);
172         handlePendingException(thread, true);
173         return verifyObject(getAndClearObjectResult(thread));
174     }
175 
176     /**
177      * Attempts to refill the current thread's TLAB and retries the allocation.
178      *
179      * @param intArrayHub the hub for {@code int[].class}
180      * @param sizeInBytes the size of the allocation
181      * @param log specifies if logging is enabled
182      *
183      * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
184      *         the operation was unsuccessful
185      */
refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log)186     static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
187         // If G1 is enabled, the "eden" allocation space is not the same always
188         // and therefore we have to go to slowpath to allocate a new TLAB.
189         if (useG1GC(INJECTED_VMCONFIG)) {
190             return WordFactory.zero();
191         }
192         if (!useTLAB(INJECTED_VMCONFIG)) {
193             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
194         }
195         Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
196         int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
197 
198         Word top = readTlabTop(thread);
199         Word end = readTlabEnd(thread);
200 
201         // calculate amount of free space
202         long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
203 
204         if (log) {
205             printf("refillTLAB: thread=%p\n", thread.rawValue());
206             printf("refillTLAB: top=%p\n", top.rawValue());
207             printf("refillTLAB: end=%p\n", end.rawValue());
208             printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
209         }
210 
211         long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
212 
213         // Retain TLAB and allocate object in shared space if
214         // the amount free in the TLAB is too large to discard.
215         Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
216         if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
217             if (tlabStats(INJECTED_VMCONFIG)) {
218                 // increment number of refills
219                 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
220                 if (log) {
221                     printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
222                 }
223                 // accumulate wastage
224                 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
225                 if (log) {
226                     printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
227                 }
228                 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
229             }
230 
231             // if TLAB is currently allocated (top or end != null) then
232             // fill [top, end + alignment_reserve) with array object
233             if (top.notEqual(0)) {
234                 int headerSize = getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int);
235                 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
236                 // an int
237                 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
238                 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
239                 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
240 
241                 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
242                 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
243                 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
244             }
245 
246             // refill the TLAB with an eden allocation
247             Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
248             Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
249             // allocate new TLAB, address returned in top
250             top = edenAllocate(tlabRefillSizeInBytes, log);
251             if (top.notEqual(0)) {
252                 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
253                 initializeTlab(thread, top, end);
254 
255                 return NewInstanceStub.allocate(thread, sizeInBytes);
256             } else {
257                 return WordFactory.zero();
258             }
259         } else {
260             // Retain TLAB
261             Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
262             thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
263             if (log) {
264                 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
265             }
266 
267             if (tlabStats(INJECTED_VMCONFIG)) {
268                 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
269                                 TLAB_SLOW_ALLOCATIONS_LOCATION);
270             }
271 
272             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
273         }
274     }
275 
276     /**
277      * Attempts to allocate a chunk of memory from Eden space.
278      *
279      * @param sizeInBytes the size of the chunk to allocate
280      * @param log specifies if logging is enabled
281      * @return the allocated chunk or {@link WordFactory#zero()} if allocation fails
282      */
edenAllocate(Word sizeInBytes, boolean log)283     public static Word edenAllocate(Word sizeInBytes, boolean log) {
284         final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
285         final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
286 
287         Word heapTopAddress = WordFactory.unsigned(heapTopRawAddress);
288         Word heapEndAddress = WordFactory.unsigned(heapEndRawAddress);
289 
290         while (true) {
291             Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
292             Word newHeapTop = heapTop.add(sizeInBytes);
293             if (newHeapTop.belowOrEqual(heapTop)) {
294                 return WordFactory.zero();
295             }
296 
297             Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
298             if (newHeapTop.aboveThan(heapEnd)) {
299                 return WordFactory.zero();
300             }
301             if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
302                 return heapTop;
303             }
304         }
305     }
306 
307     @Fold
forceSlowPath(OptionValues options)308     static boolean forceSlowPath(OptionValues options) {
309         return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
310     }
311 
312     public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
313 
314     @NodeIntrinsic(StubForeignCallNode.class)
newInstanceC(@onstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub)315     public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
316 }
317