1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// See malloc.h for overview. 6// 7// TODO(rsc): double-check stats. 8 9package runtime 10#include <stddef.h> 11#include <errno.h> 12#include <stdlib.h> 13#include "go-alloc.h" 14#include "runtime.h" 15#include "arch.h" 16#include "malloc.h" 17#include "interface.h" 18#include "go-type.h" 19 20// Map gccgo field names to gc field names. 21// Eface aka __go_empty_interface. 22#define type __type_descriptor 23// Type aka __go_type_descriptor 24#define kind __code 25#define string __reflection 26#define KindPtr GO_PTR 27#define KindNoPointers GO_NO_POINTERS 28#define kindMask GO_CODE_MASK 29 30// GCCGO SPECIFIC CHANGE 31// 32// There is a long comment in runtime_mallocinit about where to put the heap 33// on a 64-bit system. It makes assumptions that are not valid on linux/arm64 34// -- it assumes user space can choose the lower 47 bits of a pointer, but on 35// linux/arm64 we can only choose the lower 39 bits. This means the heap is 36// roughly a quarter of the available address space and we cannot choose a bit 37// pattern that all pointers will have -- luckily the GC is mostly precise 38// these days so this doesn't matter all that much. The kernel (as of 3.13) 39// will allocate address space starting either down from 0x7fffffffff or up 40// from 0x2000000000, so we put the heap roughly in the middle of these two 41// addresses to minimize the chance that a non-heap allocation will get in the 42// way of the heap. 43// 44// This all means that there isn't much point in trying 256 different 45// locations for the heap on such systems. 46#ifdef __aarch64__ 47#define HeapBase(i) ((void*)(uintptr)(0x40ULL<<32)) 48#define HeapBaseOptions 1 49#else 50#define HeapBase(i) ((void*)(uintptr)(i<<40|0x00c0ULL<<32)) 51#define HeapBaseOptions 0x80 52#endif 53// END GCCGO SPECIFIC CHANGE 54 55// Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K. 56MHeap runtime_mheap; 57MStats mstats; 58 59int32 runtime_checking; 60 61extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go 62 63extern volatile intgo runtime_MemProfileRate 64 __asm__ (GOSYM_PREFIX "runtime.MemProfileRate"); 65 66static MSpan* largealloc(uint32, uintptr*); 67static void runtime_profilealloc(void *v, uintptr size); 68static void settype(MSpan *s, void *v, uintptr typ); 69 70// Allocate an object of at least size bytes. 71// Small objects are allocated from the per-thread cache's free lists. 72// Large objects (> 32 kB) are allocated straight from the heap. 73// If the block will be freed with runtime_free(), typ must be 0. 74void* 75runtime_mallocgc(uintptr size, uintptr typ, uint32 flag) 76{ 77 M *m; 78 G *g; 79 int32 sizeclass; 80 uintptr tinysize, size1; 81 intgo rate; 82 MCache *c; 83 MSpan *s; 84 MLink *v, *next; 85 byte *tiny; 86 bool incallback; 87 88 if(size == 0) { 89 // All 0-length allocations use this pointer. 90 // The language does not require the allocations to 91 // have distinct values. 92 return &runtime_zerobase; 93 } 94 95 m = runtime_m(); 96 g = runtime_g(); 97 98 incallback = false; 99 if(m->mcache == nil && g->ncgo > 0) { 100 // For gccgo this case can occur when a cgo or SWIG function 101 // has an interface return type and the function 102 // returns a non-pointer, so memory allocation occurs 103 // after syscall.Cgocall but before syscall.CgocallDone. 104 // We treat it as a callback. 105 runtime_exitsyscall(); 106 m = runtime_m(); 107 incallback = true; 108 flag |= FlagNoInvokeGC; 109 } 110 111 if(runtime_gcwaiting() && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC)) { 112 runtime_gosched(); 113 m = runtime_m(); 114 } 115 if(m->mallocing) 116 runtime_throw("malloc/free - deadlock"); 117 // Disable preemption during settype. 118 // We can not use m->mallocing for this, because settype calls mallocgc. 119 m->locks++; 120 m->mallocing = 1; 121 122 if(DebugTypeAtBlockEnd) 123 size += sizeof(uintptr); 124 125 c = m->mcache; 126 if(!runtime_debug.efence && size <= MaxSmallSize) { 127 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize) { 128 // Tiny allocator. 129 // 130 // Tiny allocator combines several tiny allocation requests 131 // into a single memory block. The resulting memory block 132 // is freed when all subobjects are unreachable. The subobjects 133 // must be FlagNoScan (don't have pointers), this ensures that 134 // the amount of potentially wasted memory is bounded. 135 // 136 // Size of the memory block used for combining (TinySize) is tunable. 137 // Current setting is 16 bytes, which relates to 2x worst case memory 138 // wastage (when all but one subobjects are unreachable). 139 // 8 bytes would result in no wastage at all, but provides less 140 // opportunities for combining. 141 // 32 bytes provides more opportunities for combining, 142 // but can lead to 4x worst case wastage. 143 // The best case winning is 8x regardless of block size. 144 // 145 // Objects obtained from tiny allocator must not be freed explicitly. 146 // So when an object will be freed explicitly, we ensure that 147 // its size >= TinySize. 148 // 149 // SetFinalizer has a special case for objects potentially coming 150 // from tiny allocator, it such case it allows to set finalizers 151 // for an inner byte of a memory block. 152 // 153 // The main targets of tiny allocator are small strings and 154 // standalone escaping variables. On a json benchmark 155 // the allocator reduces number of allocations by ~12% and 156 // reduces heap size by ~20%. 157 158 tinysize = c->tinysize; 159 if(size <= tinysize) { 160 tiny = c->tiny; 161 // Align tiny pointer for required (conservative) alignment. 162 if((size&7) == 0) 163 tiny = (byte*)ROUND((uintptr)tiny, 8); 164 else if((size&3) == 0) 165 tiny = (byte*)ROUND((uintptr)tiny, 4); 166 else if((size&1) == 0) 167 tiny = (byte*)ROUND((uintptr)tiny, 2); 168 size1 = size + (tiny - c->tiny); 169 if(size1 <= tinysize) { 170 // The object fits into existing tiny block. 171 v = (MLink*)tiny; 172 c->tiny += size1; 173 c->tinysize -= size1; 174 m->mallocing = 0; 175 m->locks--; 176 if(incallback) 177 runtime_entersyscall(); 178 return v; 179 } 180 } 181 // Allocate a new TinySize block. 182 s = c->alloc[TinySizeClass]; 183 if(s->freelist == nil) 184 s = runtime_MCache_Refill(c, TinySizeClass); 185 v = s->freelist; 186 next = v->next; 187 s->freelist = next; 188 s->ref++; 189 if(next != nil) // prefetching nil leads to a DTLB miss 190 PREFETCH(next); 191 ((uint64*)v)[0] = 0; 192 ((uint64*)v)[1] = 0; 193 // See if we need to replace the existing tiny block with the new one 194 // based on amount of remaining free space. 195 if(TinySize-size > tinysize) { 196 c->tiny = (byte*)v + size; 197 c->tinysize = TinySize - size; 198 } 199 size = TinySize; 200 goto done; 201 } 202 // Allocate from mcache free lists. 203 // Inlined version of SizeToClass(). 204 if(size <= 1024-8) 205 sizeclass = runtime_size_to_class8[(size+7)>>3]; 206 else 207 sizeclass = runtime_size_to_class128[(size-1024+127) >> 7]; 208 size = runtime_class_to_size[sizeclass]; 209 s = c->alloc[sizeclass]; 210 if(s->freelist == nil) 211 s = runtime_MCache_Refill(c, sizeclass); 212 v = s->freelist; 213 next = v->next; 214 s->freelist = next; 215 s->ref++; 216 if(next != nil) // prefetching nil leads to a DTLB miss 217 PREFETCH(next); 218 if(!(flag & FlagNoZero)) { 219 v->next = nil; 220 // block is zeroed iff second word is zero ... 221 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0) 222 runtime_memclr((byte*)v, size); 223 } 224 done: 225 c->local_cachealloc += size; 226 } else { 227 // Allocate directly from heap. 228 s = largealloc(flag, &size); 229 v = (void*)(s->start << PageShift); 230 } 231 232 if(flag & FlagNoGC) 233 runtime_marknogc(v); 234 else if(!(flag & FlagNoScan)) 235 runtime_markscan(v); 236 237 if(DebugTypeAtBlockEnd) 238 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ; 239 240 m->mallocing = 0; 241 // TODO: save type even if FlagNoScan? Potentially expensive but might help 242 // heap profiling/tracing. 243 if(UseSpanType && !(flag & FlagNoScan) && typ != 0) 244 settype(s, v, typ); 245 246 if(runtime_debug.allocfreetrace) 247 runtime_tracealloc(v, size, typ); 248 249 if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) { 250 if(size < (uintptr)rate && size < (uintptr)(uint32)c->next_sample) 251 c->next_sample -= size; 252 else 253 runtime_profilealloc(v, size); 254 } 255 256 m->locks--; 257 258 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) 259 runtime_gc(0); 260 261 if(incallback) 262 runtime_entersyscall(); 263 264 return v; 265} 266 267static MSpan* 268largealloc(uint32 flag, uintptr *sizep) 269{ 270 uintptr npages, size; 271 MSpan *s; 272 void *v; 273 274 // Allocate directly from heap. 275 size = *sizep; 276 if(size + PageSize < size) 277 runtime_throw("out of memory"); 278 npages = size >> PageShift; 279 if((size & PageMask) != 0) 280 npages++; 281 s = runtime_MHeap_Alloc(&runtime_mheap, npages, 0, 1, !(flag & FlagNoZero)); 282 if(s == nil) 283 runtime_throw("out of memory"); 284 s->limit = (byte*)(s->start<<PageShift) + size; 285 *sizep = npages<<PageShift; 286 v = (void*)(s->start << PageShift); 287 // setup for mark sweep 288 runtime_markspan(v, 0, 0, true); 289 return s; 290} 291 292static void 293runtime_profilealloc(void *v, uintptr size) 294{ 295 uintptr rate; 296 int32 next; 297 MCache *c; 298 299 c = runtime_m()->mcache; 300 rate = runtime_MemProfileRate; 301 if(size < rate) { 302 // pick next profile time 303 // If you change this, also change allocmcache. 304 if(rate > 0x3fffffff) // make 2*rate not overflow 305 rate = 0x3fffffff; 306 next = runtime_fastrand1() % (2*rate); 307 // Subtract the "remainder" of the current allocation. 308 // Otherwise objects that are close in size to sampling rate 309 // will be under-sampled, because we consistently discard this remainder. 310 next -= (size - c->next_sample); 311 if(next < 0) 312 next = 0; 313 c->next_sample = next; 314 } 315 runtime_MProf_Malloc(v, size); 316} 317 318void* 319__go_alloc(uintptr size) 320{ 321 return runtime_mallocgc(size, 0, FlagNoInvokeGC); 322} 323 324// Free the object whose base pointer is v. 325void 326__go_free(void *v) 327{ 328 M *m; 329 int32 sizeclass; 330 MSpan *s; 331 MCache *c; 332 uintptr size; 333 334 if(v == nil) 335 return; 336 337 // If you change this also change mgc0.c:/^sweep, 338 // which has a copy of the guts of free. 339 340 m = runtime_m(); 341 if(m->mallocing) 342 runtime_throw("malloc/free - deadlock"); 343 m->mallocing = 1; 344 345 if(!runtime_mlookup(v, nil, nil, &s)) { 346 runtime_printf("free %p: not an allocated block\n", v); 347 runtime_throw("free runtime_mlookup"); 348 } 349 size = s->elemsize; 350 sizeclass = s->sizeclass; 351 // Objects that are smaller than TinySize can be allocated using tiny alloc, 352 // if then such object is combined with an object with finalizer, we will crash. 353 if(size < TinySize) 354 runtime_throw("freeing too small block"); 355 356 if(runtime_debug.allocfreetrace) 357 runtime_tracefree(v, size); 358 359 // Ensure that the span is swept. 360 // If we free into an unswept span, we will corrupt GC bitmaps. 361 runtime_MSpan_EnsureSwept(s); 362 363 if(s->specials != nil) 364 runtime_freeallspecials(s, v, size); 365 366 c = m->mcache; 367 if(sizeclass == 0) { 368 // Large object. 369 s->needzero = 1; 370 // Must mark v freed before calling unmarkspan and MHeap_Free: 371 // they might coalesce v into other spans and change the bitmap further. 372 runtime_markfreed(v); 373 runtime_unmarkspan(v, 1<<PageShift); 374 // NOTE(rsc,dvyukov): The original implementation of efence 375 // in CL 22060046 used SysFree instead of SysFault, so that 376 // the operating system would eventually give the memory 377 // back to us again, so that an efence program could run 378 // longer without running out of memory. Unfortunately, 379 // calling SysFree here without any kind of adjustment of the 380 // heap data structures means that when the memory does 381 // come back to us, we have the wrong metadata for it, either in 382 // the MSpan structures or in the garbage collection bitmap. 383 // Using SysFault here means that the program will run out of 384 // memory fairly quickly in efence mode, but at least it won't 385 // have mysterious crashes due to confused memory reuse. 386 // It should be possible to switch back to SysFree if we also 387 // implement and then call some kind of MHeap_DeleteSpan. 388 if(runtime_debug.efence) 389 runtime_SysFault((void*)(s->start<<PageShift), size); 390 else 391 runtime_MHeap_Free(&runtime_mheap, s, 1); 392 c->local_nlargefree++; 393 c->local_largefree += size; 394 } else { 395 // Small object. 396 if(size > 2*sizeof(uintptr)) 397 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed" 398 else if(size > sizeof(uintptr)) 399 ((uintptr*)v)[1] = 0; 400 // Must mark v freed before calling MCache_Free: 401 // it might coalesce v and other blocks into a bigger span 402 // and change the bitmap further. 403 c->local_nsmallfree[sizeclass]++; 404 c->local_cachealloc -= size; 405 if(c->alloc[sizeclass] == s) { 406 // We own the span, so we can just add v to the freelist 407 runtime_markfreed(v); 408 ((MLink*)v)->next = s->freelist; 409 s->freelist = v; 410 s->ref--; 411 } else { 412 // Someone else owns this span. Add to free queue. 413 runtime_MCache_Free(c, v, sizeclass, size); 414 } 415 } 416 m->mallocing = 0; 417} 418 419int32 420runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp) 421{ 422 M *m; 423 uintptr n, i; 424 byte *p; 425 MSpan *s; 426 427 m = runtime_m(); 428 429 m->mcache->local_nlookup++; 430 if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) { 431 // purge cache stats to prevent overflow 432 runtime_lock(&runtime_mheap.lock); 433 runtime_purgecachedstats(m->mcache); 434 runtime_unlock(&runtime_mheap.lock); 435 } 436 437 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v); 438 if(sp) 439 *sp = s; 440 if(s == nil) { 441 runtime_checkfreed(v, 1); 442 if(base) 443 *base = nil; 444 if(size) 445 *size = 0; 446 return 0; 447 } 448 449 p = (byte*)((uintptr)s->start<<PageShift); 450 if(s->sizeclass == 0) { 451 // Large object. 452 if(base) 453 *base = p; 454 if(size) 455 *size = s->npages<<PageShift; 456 return 1; 457 } 458 459 n = s->elemsize; 460 if(base) { 461 i = ((byte*)v - p)/n; 462 *base = p + i*n; 463 } 464 if(size) 465 *size = n; 466 467 return 1; 468} 469 470void 471runtime_purgecachedstats(MCache *c) 472{ 473 MHeap *h; 474 int32 i; 475 476 // Protected by either heap or GC lock. 477 h = &runtime_mheap; 478 mstats.heap_alloc += c->local_cachealloc; 479 c->local_cachealloc = 0; 480 mstats.nlookup += c->local_nlookup; 481 c->local_nlookup = 0; 482 h->largefree += c->local_largefree; 483 c->local_largefree = 0; 484 h->nlargefree += c->local_nlargefree; 485 c->local_nlargefree = 0; 486 for(i=0; i<(int32)nelem(c->local_nsmallfree); i++) { 487 h->nsmallfree[i] += c->local_nsmallfree[i]; 488 c->local_nsmallfree[i] = 0; 489 } 490} 491 492extern uintptr runtime_sizeof_C_MStats 493 __asm__ (GOSYM_PREFIX "runtime.Sizeof_C_MStats"); 494 495// Size of the trailing by_size array differs between Go and C, 496// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. 497// sizeof_C_MStats is what C thinks about size of Go struct. 498 499// Initialized in mallocinit because it's defined in go/runtime/mem.go. 500 501#define MaxArena32 (2U<<30) 502 503void 504runtime_mallocinit(void) 505{ 506 byte *p, *p1; 507 uintptr arena_size, bitmap_size, spans_size, p_size; 508 extern byte _end[]; 509 uintptr limit; 510 uint64 i; 511 bool reserved; 512 513 runtime_sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeof(mstats.by_size[0]); 514 515 p = nil; 516 p_size = 0; 517 arena_size = 0; 518 bitmap_size = 0; 519 spans_size = 0; 520 reserved = false; 521 522 // for 64-bit build 523 USED(p); 524 USED(p_size); 525 USED(arena_size); 526 USED(bitmap_size); 527 USED(spans_size); 528 529 runtime_InitSizes(); 530 531 if(runtime_class_to_size[TinySizeClass] != TinySize) 532 runtime_throw("bad TinySizeClass"); 533 534 // limit = runtime_memlimit(); 535 // See https://code.google.com/p/go/issues/detail?id=5049 536 // TODO(rsc): Fix after 1.1. 537 limit = 0; 538 539 // Set up the allocation arena, a contiguous area of memory where 540 // allocated data will be found. The arena begins with a bitmap large 541 // enough to hold 4 bits per allocated word. 542 if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { 543 // On a 64-bit machine, allocate from a single contiguous reservation. 544 // 128 GB (MaxMem) should be big enough for now. 545 // 546 // The code will work with the reservation at any address, but ask 547 // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). 548 // Allocating a 128 GB region takes away 37 bits, and the amd64 549 // doesn't let us choose the top 17 bits, so that leaves the 11 bits 550 // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means 551 // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. 552 // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid 553 // UTF-8 sequences, and they are otherwise as far away from 554 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 555 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors 556 // on OS X during thread allocations. 0x00c0 causes conflicts with 557 // AddressSanitizer which reserves all memory up to 0x0100. 558 // These choices are both for debuggability and to reduce the 559 // odds of the conservative garbage collector not collecting memory 560 // because some non-pointer block of memory had a bit pattern 561 // that matched a memory address. 562 // 563 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) 564 // but it hardly matters: e0 00 is not valid UTF-8 either. 565 // 566 // If this fails we fall back to the 32 bit memory mechanism 567 arena_size = MaxMem; 568 bitmap_size = arena_size / (sizeof(void*)*8/4); 569 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]); 570 spans_size = ROUND(spans_size, PageSize); 571 for(i = 0; i < HeapBaseOptions; i++) { 572 p = HeapBase(i); 573 p_size = bitmap_size + spans_size + arena_size + PageSize; 574 p = runtime_SysReserve(p, p_size, &reserved); 575 if(p != nil) 576 break; 577 } 578 } 579 if (p == nil) { 580 // On a 32-bit machine, we can't typically get away 581 // with a giant virtual address space reservation. 582 // Instead we map the memory information bitmap 583 // immediately after the data segment, large enough 584 // to handle another 2GB of mappings (256 MB), 585 // along with a reservation for another 512 MB of memory. 586 // When that gets used up, we'll start asking the kernel 587 // for any memory anywhere and hope it's in the 2GB 588 // following the bitmap (presumably the executable begins 589 // near the bottom of memory, so we'll have to use up 590 // most of memory before the kernel resorts to giving out 591 // memory before the beginning of the text segment). 592 // 593 // Alternatively we could reserve 512 MB bitmap, enough 594 // for 4GB of mappings, and then accept any memory the 595 // kernel threw at us, but normally that's a waste of 512 MB 596 // of address space, which is probably too much in a 32-bit world. 597 bitmap_size = MaxArena32 / (sizeof(void*)*8/4); 598 arena_size = 512<<20; 599 spans_size = MaxArena32 / PageSize * sizeof(runtime_mheap.spans[0]); 600 if(limit > 0 && arena_size+bitmap_size+spans_size > limit) { 601 bitmap_size = (limit / 9) & ~((1<<PageShift) - 1); 602 arena_size = bitmap_size * 8; 603 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]); 604 } 605 spans_size = ROUND(spans_size, PageSize); 606 607 // SysReserve treats the address we ask for, end, as a hint, 608 // not as an absolute requirement. If we ask for the end 609 // of the data segment but the operating system requires 610 // a little more space before we can start allocating, it will 611 // give out a slightly higher pointer. Except QEMU, which 612 // is buggy, as usual: it won't adjust the pointer upward. 613 // So adjust it upward a little bit ourselves: 1/4 MB to get 614 // away from the running binary image and then round up 615 // to a MB boundary. 616 p = (byte*)ROUND((uintptr)_end + (1<<18), 1<<20); 617 p_size = bitmap_size + spans_size + arena_size + PageSize; 618 p = runtime_SysReserve(p, p_size, &reserved); 619 if(p == nil) 620 runtime_throw("runtime: cannot reserve arena virtual address space"); 621 } 622 623 // PageSize can be larger than OS definition of page size, 624 // so SysReserve can give us a PageSize-unaligned pointer. 625 // To overcome this we ask for PageSize more and round up the pointer. 626 p1 = (byte*)ROUND((uintptr)p, PageSize); 627 628 runtime_mheap.spans = (MSpan**)p1; 629 runtime_mheap.bitmap = p1 + spans_size; 630 runtime_mheap.arena_start = p1 + spans_size + bitmap_size; 631 runtime_mheap.arena_used = runtime_mheap.arena_start; 632 runtime_mheap.arena_end = p + p_size; 633 runtime_mheap.arena_reserved = reserved; 634 635 if(((uintptr)runtime_mheap.arena_start & (PageSize-1)) != 0) 636 runtime_throw("misrounded allocation in mallocinit"); 637 638 // Initialize the rest of the allocator. 639 runtime_MHeap_Init(&runtime_mheap); 640 runtime_m()->mcache = runtime_allocmcache(); 641 642 // See if it works. 643 runtime_free(runtime_malloc(TinySize)); 644} 645 646void* 647runtime_MHeap_SysAlloc(MHeap *h, uintptr n) 648{ 649 byte *p, *p_end; 650 uintptr p_size; 651 bool reserved; 652 653 654 if(n > (uintptr)(h->arena_end - h->arena_used)) { 655 // We are in 32-bit mode, maybe we didn't use all possible address space yet. 656 // Reserve some more space. 657 byte *new_end; 658 659 p_size = ROUND(n + PageSize, 256<<20); 660 new_end = h->arena_end + p_size; 661 if(new_end <= h->arena_start + MaxArena32) { 662 // TODO: It would be bad if part of the arena 663 // is reserved and part is not. 664 p = runtime_SysReserve(h->arena_end, p_size, &reserved); 665 if(p == h->arena_end) { 666 h->arena_end = new_end; 667 h->arena_reserved = reserved; 668 } 669 else if(p+p_size <= h->arena_start + MaxArena32) { 670 // Keep everything page-aligned. 671 // Our pages are bigger than hardware pages. 672 h->arena_end = p+p_size; 673 h->arena_used = p + (-(uintptr)p&(PageSize-1)); 674 h->arena_reserved = reserved; 675 } else { 676 uint64 stat; 677 stat = 0; 678 runtime_SysFree(p, p_size, &stat); 679 } 680 } 681 } 682 if(n <= (uintptr)(h->arena_end - h->arena_used)) { 683 // Keep taking from our reservation. 684 p = h->arena_used; 685 runtime_SysMap(p, n, h->arena_reserved, &mstats.heap_sys); 686 h->arena_used += n; 687 runtime_MHeap_MapBits(h); 688 runtime_MHeap_MapSpans(h); 689 690 if(((uintptr)p & (PageSize-1)) != 0) 691 runtime_throw("misrounded allocation in MHeap_SysAlloc"); 692 return p; 693 } 694 695 // If using 64-bit, our reservation is all we have. 696 if((uintptr)(h->arena_end - h->arena_start) >= MaxArena32) 697 return nil; 698 699 // On 32-bit, once the reservation is gone we can 700 // try to get memory at a location chosen by the OS 701 // and hope that it is in the range we allocated bitmap for. 702 p_size = ROUND(n, PageSize) + PageSize; 703 p = runtime_SysAlloc(p_size, &mstats.heap_sys); 704 if(p == nil) 705 return nil; 706 707 if(p < h->arena_start || (uintptr)(p+p_size - h->arena_start) >= MaxArena32) { 708 runtime_printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n", 709 p, h->arena_start, h->arena_start+MaxArena32); 710 runtime_SysFree(p, p_size, &mstats.heap_sys); 711 return nil; 712 } 713 714 p_end = p + p_size; 715 p += -(uintptr)p & (PageSize-1); 716 if(p+n > h->arena_used) { 717 h->arena_used = p+n; 718 if(p_end > h->arena_end) 719 h->arena_end = p_end; 720 runtime_MHeap_MapBits(h); 721 runtime_MHeap_MapSpans(h); 722 } 723 724 if(((uintptr)p & (PageSize-1)) != 0) 725 runtime_throw("misrounded allocation in MHeap_SysAlloc"); 726 return p; 727} 728 729static struct 730{ 731 Lock lock; 732 byte* pos; 733 byte* end; 734} persistent; 735 736enum 737{ 738 PersistentAllocChunk = 256<<10, 739 PersistentAllocMaxBlock = 64<<10, // VM reservation granularity is 64K on windows 740}; 741 742// Wrapper around SysAlloc that can allocate small chunks. 743// There is no associated free operation. 744// Intended for things like function/type/debug-related persistent data. 745// If align is 0, uses default align (currently 8). 746void* 747runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat) 748{ 749 byte *p; 750 751 if(align != 0) { 752 if(align&(align-1)) 753 runtime_throw("persistentalloc: align is not a power of 2"); 754 if(align > PageSize) 755 runtime_throw("persistentalloc: align is too large"); 756 } else 757 align = 8; 758 if(size >= PersistentAllocMaxBlock) 759 return runtime_SysAlloc(size, stat); 760 runtime_lock(&persistent.lock); 761 persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align); 762 if(persistent.pos + size > persistent.end) { 763 persistent.pos = runtime_SysAlloc(PersistentAllocChunk, &mstats.other_sys); 764 if(persistent.pos == nil) { 765 runtime_unlock(&persistent.lock); 766 runtime_throw("runtime: cannot allocate memory"); 767 } 768 persistent.end = persistent.pos + PersistentAllocChunk; 769 } 770 p = persistent.pos; 771 persistent.pos += size; 772 runtime_unlock(&persistent.lock); 773 if(stat != &mstats.other_sys) { 774 // reaccount the allocation against provided stat 775 runtime_xadd64(stat, size); 776 runtime_xadd64(&mstats.other_sys, -(uint64)size); 777 } 778 return p; 779} 780 781static void 782settype(MSpan *s, void *v, uintptr typ) 783{ 784 uintptr size, ofs, j, t; 785 uintptr ntypes, nbytes2, nbytes3; 786 uintptr *data2; 787 byte *data3; 788 789 if(s->sizeclass == 0) { 790 s->types.compression = MTypes_Single; 791 s->types.data = typ; 792 return; 793 } 794 size = s->elemsize; 795 ofs = ((uintptr)v - (s->start<<PageShift)) / size; 796 797 switch(s->types.compression) { 798 case MTypes_Empty: 799 ntypes = (s->npages << PageShift) / size; 800 nbytes3 = 8*sizeof(uintptr) + 1*ntypes; 801 data3 = runtime_mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC); 802 s->types.compression = MTypes_Bytes; 803 s->types.data = (uintptr)data3; 804 ((uintptr*)data3)[1] = typ; 805 data3[8*sizeof(uintptr) + ofs] = 1; 806 break; 807 808 case MTypes_Words: 809 ((uintptr*)s->types.data)[ofs] = typ; 810 break; 811 812 case MTypes_Bytes: 813 data3 = (byte*)s->types.data; 814 for(j=1; j<8; j++) { 815 if(((uintptr*)data3)[j] == typ) { 816 break; 817 } 818 if(((uintptr*)data3)[j] == 0) { 819 ((uintptr*)data3)[j] = typ; 820 break; 821 } 822 } 823 if(j < 8) { 824 data3[8*sizeof(uintptr) + ofs] = j; 825 } else { 826 ntypes = (s->npages << PageShift) / size; 827 nbytes2 = ntypes * sizeof(uintptr); 828 data2 = runtime_mallocgc(nbytes2, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC); 829 s->types.compression = MTypes_Words; 830 s->types.data = (uintptr)data2; 831 832 // Move the contents of data3 to data2. Then deallocate data3. 833 for(j=0; j<ntypes; j++) { 834 t = data3[8*sizeof(uintptr) + j]; 835 t = ((uintptr*)data3)[t]; 836 data2[j] = t; 837 } 838 data2[ofs] = typ; 839 } 840 break; 841 } 842} 843 844uintptr 845runtime_gettype(void *v) 846{ 847 MSpan *s; 848 uintptr t, ofs; 849 byte *data; 850 851 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v); 852 if(s != nil) { 853 t = 0; 854 switch(s->types.compression) { 855 case MTypes_Empty: 856 break; 857 case MTypes_Single: 858 t = s->types.data; 859 break; 860 case MTypes_Words: 861 ofs = (uintptr)v - (s->start<<PageShift); 862 t = ((uintptr*)s->types.data)[ofs/s->elemsize]; 863 break; 864 case MTypes_Bytes: 865 ofs = (uintptr)v - (s->start<<PageShift); 866 data = (byte*)s->types.data; 867 t = data[8*sizeof(uintptr) + ofs/s->elemsize]; 868 t = ((uintptr*)data)[t]; 869 break; 870 default: 871 runtime_throw("runtime_gettype: invalid compression kind"); 872 } 873 if(0) { 874 runtime_printf("%p -> %d,%X\n", v, (int32)s->types.compression, (int64)t); 875 } 876 return t; 877 } 878 return 0; 879} 880 881// Runtime stubs. 882 883void* 884runtime_mal(uintptr n) 885{ 886 return runtime_mallocgc(n, 0, 0); 887} 888 889func new(typ *Type) (ret *uint8) { 890 ret = runtime_mallocgc(typ->__size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoScan : 0); 891} 892 893static void* 894cnew(const Type *typ, intgo n, int32 objtyp) 895{ 896 if((objtyp&(PtrSize-1)) != objtyp) 897 runtime_throw("runtime: invalid objtyp"); 898 if(n < 0 || (typ->__size > 0 && (uintptr)n > (MaxMem/typ->__size))) 899 runtime_panicstring("runtime: allocation size out of range"); 900 return runtime_mallocgc(typ->__size*n, (uintptr)typ | objtyp, typ->kind&KindNoPointers ? FlagNoScan : 0); 901} 902 903// same as runtime_new, but callable from C 904void* 905runtime_cnew(const Type *typ) 906{ 907 return cnew(typ, 1, TypeInfo_SingleObject); 908} 909 910void* 911runtime_cnewarray(const Type *typ, intgo n) 912{ 913 return cnew(typ, n, TypeInfo_Array); 914} 915 916func GC() { 917 runtime_gc(2); // force GC and do eager sweep 918} 919 920func SetFinalizer(obj Eface, finalizer Eface) { 921 byte *base; 922 uintptr size; 923 const FuncType *ft; 924 const Type *fint; 925 const PtrType *ot; 926 927 if(obj.__type_descriptor == nil) { 928 runtime_printf("runtime.SetFinalizer: first argument is nil interface\n"); 929 goto throw; 930 } 931 if((obj.__type_descriptor->kind&kindMask) != GO_PTR) { 932 runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.__type_descriptor->__reflection); 933 goto throw; 934 } 935 ot = (const PtrType*)obj.type; 936 // As an implementation detail we do not run finalizers for zero-sized objects, 937 // because we use &runtime_zerobase for all such allocations. 938 if(ot->__element_type != nil && ot->__element_type->__size == 0) 939 return; 940 // The following check is required for cases when a user passes a pointer to composite literal, 941 // but compiler makes it a pointer to global. For example: 942 // var Foo = &Object{} 943 // func main() { 944 // runtime.SetFinalizer(Foo, nil) 945 // } 946 // See issue 7656. 947 if((byte*)obj.__object < runtime_mheap.arena_start || runtime_mheap.arena_used <= (byte*)obj.__object) 948 return; 949 if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) { 950 // As an implementation detail we allow to set finalizers for an inner byte 951 // of an object if it could come from tiny alloc (see mallocgc for details). 952 if(ot->__element_type == nil || (ot->__element_type->kind&KindNoPointers) == 0 || ot->__element_type->__size >= TinySize) { 953 runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block (%p)\n", obj.__object); 954 goto throw; 955 } 956 } 957 if(finalizer.__type_descriptor != nil) { 958 runtime_createfing(); 959 if((finalizer.__type_descriptor->kind&kindMask) != GO_FUNC) 960 goto badfunc; 961 ft = (const FuncType*)finalizer.__type_descriptor; 962 if(ft->__dotdotdot || ft->__in.__count != 1) 963 goto badfunc; 964 fint = *(Type**)ft->__in.__values; 965 if(__go_type_descriptors_equal(fint, obj.__type_descriptor)) { 966 // ok - same type 967 } else if((fint->kind&kindMask) == GO_PTR && (fint->__uncommon == nil || fint->__uncommon->__name == nil || obj.type->__uncommon == nil || obj.type->__uncommon->__name == nil) && __go_type_descriptors_equal(((const PtrType*)fint)->__element_type, ((const PtrType*)obj.type)->__element_type)) { 968 // ok - not same type, but both pointers, 969 // one or the other is unnamed, and same element type, so assignable. 970 } else if((fint->kind&kindMask) == GO_INTERFACE && ((const InterfaceType*)fint)->__methods.__count == 0) { 971 // ok - satisfies empty interface 972 } else if((fint->kind&kindMask) == GO_INTERFACE && __go_convert_interface_2(fint, obj.__type_descriptor, 1) != nil) { 973 // ok - satisfies non-empty interface 974 } else 975 goto badfunc; 976 977 ot = (const PtrType*)obj.__type_descriptor; 978 if(!runtime_addfinalizer(obj.__object, *(FuncVal**)finalizer.__object, ft, ot)) { 979 runtime_printf("runtime.SetFinalizer: finalizer already set\n"); 980 goto throw; 981 } 982 } else { 983 // NOTE: asking to remove a finalizer when there currently isn't one set is OK. 984 runtime_removefinalizer(obj.__object); 985 } 986 return; 987 988badfunc: 989 runtime_printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.__type_descriptor->__reflection, *finalizer.__type_descriptor->__reflection); 990throw: 991 runtime_throw("runtime.SetFinalizer"); 992} 993