1 /* 2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.hotspot.stubs; 26 27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION; 34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION; 35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION; 36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION; 37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION; 38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult; 39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getArrayBaseOffset; 40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab; 41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; 42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize; 43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; 44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; 46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart; 47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; 48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; 49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset; 50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset; 51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords; 52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset; 53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord; 54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset; 55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement; 56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset; 57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset; 58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats; 59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode; 60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useFastTLABRefill; 61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC; 62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB; 63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize; 64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop; 65 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException; 66 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor; 67 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf; 68 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject; 69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; 70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 71 72 import org.graalvm.compiler.api.replacements.Fold; 73 import org.graalvm.compiler.api.replacements.Snippet; 74 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; 76 import org.graalvm.compiler.graph.Node.ConstantNodeParameter; 77 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 78 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; 79 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 80 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode; 81 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode; 82 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; 83 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets; 84 import org.graalvm.compiler.hotspot.word.KlassPointer; 85 import org.graalvm.compiler.nodes.ConstantNode; 86 import org.graalvm.compiler.options.OptionValues; 87 import org.graalvm.compiler.word.Word; 88 import jdk.internal.vm.compiler.word.WordFactory; 89 90 import jdk.vm.ci.code.Register; 91 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; 92 import jdk.vm.ci.meta.JavaKind; 93 94 /** 95 * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is 96 * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails. 97 * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++ 98 * runtime for to complete the allocation. 99 */ 100 public class NewInstanceStub extends SnippetStub { 101 NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage)102 public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { 103 super("newInstance", options, providers, linkage); 104 } 105 106 @Override makeConstArgs()107 protected Object[] makeConstArgs() { 108 HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class); 109 int count = method.getSignature().getParameterCount(false); 110 Object[] args = new Object[count]; 111 assert checkConstArg(1, "intArrayHub"); 112 assert checkConstArg(2, "threadRegister"); 113 assert checkConstArg(3, "options"); 114 args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null); 115 args[2] = providers.getRegisters().getThreadRegister(); 116 args[3] = options; 117 return args; 118 } 119 allocate(Word thread, int size)120 private static Word allocate(Word thread, int size) { 121 Word top = readTlabTop(thread); 122 Word end = readTlabEnd(thread); 123 Word newTop = top.add(size); 124 /* 125 * this check might lead to problems if the TLAB is within 16GB of the address space end 126 * (checked in c++ code) 127 */ 128 if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { 129 writeTlabTop(thread, newTop); 130 return top; 131 } 132 return WordFactory.zero(); 133 } 134 135 @Fold logging(OptionValues options)136 static boolean logging(OptionValues options) { 137 return StubOptions.TraceNewInstanceStub.getValue(options); 138 } 139 140 /** 141 * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to 142 * -XX:-UseTLAB). 143 * 144 * @param hub the hub of the object to be allocated 145 * @param intArrayHub the hub for {@code int[].class} 146 */ 147 @Snippet newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options)148 private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) { 149 /* 150 * The type is known to be an instance so Klass::_layout_helper is the instance size as a 151 * raw number 152 */ 153 Word thread = registerAsWord(threadRegister); 154 boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported(); 155 if (useFastTLABRefill(INJECTED_VMCONFIG) && !forceSlowPath(options) && inlineContiguousAllocationSupported && !useCMSIncrementalMode(INJECTED_VMCONFIG)) { 156 if (isInstanceKlassFullyInitialized(hub)) { 157 int sizeInBytes = readLayoutHelper(hub); 158 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options)); 159 if (memory.notEqual(0)) { 160 Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); 161 NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord); 162 return verifyObject(memory.toObject()); 163 } 164 } 165 } 166 167 if (logging(options)) { 168 printf("newInstance: calling new_instance_c\n"); 169 } 170 171 newInstanceC(NEW_INSTANCE_C, thread, hub); 172 handlePendingException(thread, true); 173 return verifyObject(getAndClearObjectResult(thread)); 174 } 175 176 /** 177 * Attempts to refill the current thread's TLAB and retries the allocation. 178 * 179 * @param intArrayHub the hub for {@code int[].class} 180 * @param sizeInBytes the size of the allocation 181 * @param log specifies if logging is enabled 182 * 183 * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if 184 * the operation was unsuccessful 185 */ refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log)186 static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) { 187 // If G1 is enabled, the "eden" allocation space is not the same always 188 // and therefore we have to go to slowpath to allocate a new TLAB. 189 if (useG1GC(INJECTED_VMCONFIG)) { 190 return WordFactory.zero(); 191 } 192 if (!useTLAB(INJECTED_VMCONFIG)) { 193 return edenAllocate(WordFactory.unsigned(sizeInBytes), log); 194 } 195 Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG)); 196 int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize(); 197 198 Word top = readTlabTop(thread); 199 Word end = readTlabEnd(thread); 200 201 // calculate amount of free space 202 long tlabFreeSpaceInBytes = end.subtract(top).rawValue(); 203 204 if (log) { 205 printf("refillTLAB: thread=%p\n", thread.rawValue()); 206 printf("refillTLAB: top=%p\n", top.rawValue()); 207 printf("refillTLAB: end=%p\n", end.rawValue()); 208 printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes); 209 } 210 211 long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize(); 212 213 // Retain TLAB and allocate object in shared space if 214 // the amount free in the TLAB is too large to discard. 215 Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION); 216 if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) { 217 if (tlabStats(INJECTED_VMCONFIG)) { 218 // increment number of refills 219 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION); 220 if (log) { 221 printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION)); 222 } 223 // accumulate wastage 224 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords; 225 if (log) { 226 printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage); 227 } 228 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION); 229 } 230 231 // if TLAB is currently allocated (top or end != null) then 232 // fill [top, end + alignment_reserve) with array object 233 if (top.notEqual(0)) { 234 int headerSize = getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int); 235 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in 236 // an int 237 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2; 238 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts; 239 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null); 240 241 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION); 242 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue(); 243 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION); 244 } 245 246 // refill the TLAB with an eden allocation 247 Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION); 248 Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize()); 249 // allocate new TLAB, address returned in top 250 top = edenAllocate(tlabRefillSizeInBytes, log); 251 if (top.notEqual(0)) { 252 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes)); 253 initializeTlab(thread, top, end); 254 255 return NewInstanceStub.allocate(thread, sizeInBytes); 256 } else { 257 return WordFactory.zero(); 258 } 259 } else { 260 // Retain TLAB 261 Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG)); 262 thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION); 263 if (log) { 264 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue()); 265 } 266 267 if (tlabStats(INJECTED_VMCONFIG)) { 268 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1, 269 TLAB_SLOW_ALLOCATIONS_LOCATION); 270 } 271 272 return edenAllocate(WordFactory.unsigned(sizeInBytes), log); 273 } 274 } 275 276 /** 277 * Attempts to allocate a chunk of memory from Eden space. 278 * 279 * @param sizeInBytes the size of the chunk to allocate 280 * @param log specifies if logging is enabled 281 * @return the allocated chunk or {@link WordFactory#zero()} if allocation fails 282 */ edenAllocate(Word sizeInBytes, boolean log)283 public static Word edenAllocate(Word sizeInBytes, boolean log) { 284 final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress(); 285 final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress(); 286 287 Word heapTopAddress = WordFactory.unsigned(heapTopRawAddress); 288 Word heapEndAddress = WordFactory.unsigned(heapEndRawAddress); 289 290 while (true) { 291 Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION); 292 Word newHeapTop = heapTop.add(sizeInBytes); 293 if (newHeapTop.belowOrEqual(heapTop)) { 294 return WordFactory.zero(); 295 } 296 297 Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION); 298 if (newHeapTop.aboveThan(heapEnd)) { 299 return WordFactory.zero(); 300 } 301 if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) { 302 return heapTop; 303 } 304 } 305 } 306 307 @Fold forceSlowPath(OptionValues options)308 static boolean forceSlowPath(OptionValues options) { 309 return StubOptions.ForceUseOfNewInstanceStub.getValue(options); 310 } 311 312 public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class); 313 314 @NodeIntrinsic(StubForeignCallNode.class) newInstanceC(@onstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub)315 public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub); 316 } 317