1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import ( 8 "runtime/internal/math" 9 "runtime/internal/sys" 10 "unsafe" 11) 12 13// For gccgo, use go:linkname to export compiler-called functions. 14// 15//go:linkname panicmakeslicelen 16//go:linkname panicmakeslicecap 17//go:linkname makeslice 18//go:linkname makeslice64 19//go:linkname growslice 20//go:linkname slicecopy 21//go:linkname slicestringcopy 22 23type slice struct { 24 array unsafe.Pointer 25 len int 26 cap int 27} 28 29// A notInHeapSlice is a slice backed by go:notinheap memory. 30type notInHeapSlice struct { 31 array *notInHeap 32 len int 33 cap int 34} 35 36func panicmakeslicelen() { 37 panic(errorString("makeslice: len out of range")) 38} 39 40func panicmakeslicecap() { 41 panic(errorString("makeslice: cap out of range")) 42} 43 44func makeslice(et *_type, len, cap int) unsafe.Pointer { 45 mem, overflow := math.MulUintptr(et.size, uintptr(cap)) 46 if overflow || mem > maxAlloc || len < 0 || len > cap { 47 // NOTE: Produce a 'len out of range' error instead of a 48 // 'cap out of range' error when someone does make([]T, bignumber). 49 // 'cap out of range' is true too, but since the cap is only being 50 // supplied implicitly, saying len is clearer. 51 // See golang.org/issue/4085. 52 mem, overflow := math.MulUintptr(et.size, uintptr(len)) 53 if overflow || mem > maxAlloc || len < 0 { 54 panicmakeslicelen() 55 } 56 panicmakeslicecap() 57 } 58 59 return mallocgc(mem, et, true) 60} 61 62func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer { 63 len := int(len64) 64 if int64(len) != len64 { 65 panicmakeslicelen() 66 } 67 68 cap := int(cap64) 69 if int64(cap) != cap64 { 70 panicmakeslicecap() 71 } 72 73 return makeslice(et, len, cap) 74} 75 76// growslice handles slice growth during append. 77// It is passed the slice element type, the old slice, and the desired new minimum capacity, 78// and it returns a new slice with at least that capacity, with the old data 79// copied into it. 80// The new slice's length is set to the requested capacity. 81func growslice(et *_type, oldarray unsafe.Pointer, oldlen, oldcap, cap int) slice { 82 if raceenabled { 83 callerpc := getcallerpc() 84 racereadrangepc(oldarray, uintptr(oldlen*int(et.size)), callerpc, funcPC(growslice)) 85 } 86 if msanenabled { 87 msanread(oldarray, uintptr(oldlen*int(et.size))) 88 } 89 90 if cap < oldcap { 91 panic(errorString("growslice: cap out of range")) 92 } 93 94 if et.size == 0 { 95 // append should not create a slice with nil pointer but non-zero len. 96 // We assume that append doesn't need to preserve oldarray in this case. 97 return slice{unsafe.Pointer(&zerobase), cap, cap} 98 } 99 100 newcap := oldcap 101 doublecap := newcap + newcap 102 if cap > doublecap { 103 newcap = cap 104 } else { 105 if oldlen < 1024 { 106 newcap = doublecap 107 } else { 108 // Check 0 < newcap to detect overflow 109 // and prevent an infinite loop. 110 for 0 < newcap && newcap < cap { 111 newcap += newcap / 4 112 } 113 // Set newcap to the requested cap when 114 // the newcap calculation overflowed. 115 if newcap <= 0 { 116 newcap = cap 117 } 118 } 119 } 120 121 var overflow bool 122 var lenmem, newlenmem, capmem uintptr 123 // Specialize for common values of et.size. 124 // For 1 we don't need any division/multiplication. 125 // For sys.PtrSize, compiler will optimize division/multiplication into a shift by a constant. 126 // For powers of 2, use a variable shift. 127 switch { 128 case et.size == 1: 129 lenmem = uintptr(oldlen) 130 newlenmem = uintptr(cap) 131 capmem = roundupsize(uintptr(newcap)) 132 overflow = uintptr(newcap) > maxAlloc 133 newcap = int(capmem) 134 case et.size == sys.PtrSize: 135 lenmem = uintptr(oldlen) * sys.PtrSize 136 newlenmem = uintptr(cap) * sys.PtrSize 137 capmem = roundupsize(uintptr(newcap) * sys.PtrSize) 138 overflow = uintptr(newcap) > maxAlloc/sys.PtrSize 139 newcap = int(capmem / sys.PtrSize) 140 case isPowerOfTwo(et.size): 141 var shift uintptr 142 if sys.PtrSize == 8 { 143 // Mask shift for better code generation. 144 shift = uintptr(sys.Ctz64(uint64(et.size))) & 63 145 } else { 146 shift = uintptr(sys.Ctz32(uint32(et.size))) & 31 147 } 148 lenmem = uintptr(oldlen) << shift 149 newlenmem = uintptr(cap) << shift 150 capmem = roundupsize(uintptr(newcap) << shift) 151 overflow = uintptr(newcap) > (maxAlloc >> shift) 152 newcap = int(capmem >> shift) 153 default: 154 lenmem = uintptr(oldlen) * et.size 155 newlenmem = uintptr(cap) * et.size 156 capmem, overflow = math.MulUintptr(et.size, uintptr(newcap)) 157 capmem = roundupsize(capmem) 158 newcap = int(capmem / et.size) 159 } 160 161 // The check of overflow in addition to capmem > maxAlloc is needed 162 // to prevent an overflow which can be used to trigger a segfault 163 // on 32bit architectures with this example program: 164 // 165 // type T [1<<27 + 1]int64 166 // 167 // var d T 168 // var s []T 169 // 170 // func main() { 171 // s = append(s, d, d, d, d) 172 // print(len(s), "\n") 173 // } 174 if overflow || capmem > maxAlloc { 175 panic(errorString("growslice: cap out of range")) 176 } 177 178 var p unsafe.Pointer 179 if et.ptrdata == 0 { 180 p = mallocgc(capmem, nil, false) 181 // The append() that calls growslice is going to overwrite from oldlen to cap (which will be the new length). 182 // Only clear the part that will not be overwritten. 183 memclrNoHeapPointers(add(p, newlenmem), capmem-newlenmem) 184 } else { 185 // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. 186 p = mallocgc(capmem, et, true) 187 if lenmem > 0 && writeBarrier.enabled { 188 // Only shade the pointers in old.array since we know the destination slice p 189 // only contains nil pointers because it has been cleared during alloc. 190 bulkBarrierPreWriteSrcOnly(uintptr(p), uintptr(oldarray), lenmem) 191 } 192 } 193 memmove(p, oldarray, lenmem) 194 195 return slice{p, cap, newcap} 196} 197 198func isPowerOfTwo(x uintptr) bool { 199 return x&(x-1) == 0 200} 201 202func slicecopy(to, fm slice, width uintptr) int { 203 if fm.len == 0 || to.len == 0 { 204 return 0 205 } 206 207 n := fm.len 208 if to.len < n { 209 n = to.len 210 } 211 212 if width == 0 { 213 return n 214 } 215 216 if raceenabled { 217 callerpc := getcallerpc() 218 pc := funcPC(slicecopy) 219 racewriterangepc(to.array, uintptr(n*int(width)), callerpc, pc) 220 racereadrangepc(fm.array, uintptr(n*int(width)), callerpc, pc) 221 } 222 if msanenabled { 223 msanwrite(to.array, uintptr(n*int(width))) 224 msanread(fm.array, uintptr(n*int(width))) 225 } 226 227 size := uintptr(n) * width 228 if size == 1 { // common case worth about 2x to do here 229 // TODO: is this still worth it with new memmove impl? 230 *(*byte)(to.array) = *(*byte)(fm.array) // known to be a byte pointer 231 } else { 232 memmove(to.array, fm.array, size) 233 } 234 return n 235} 236 237func slicestringcopy(to []byte, fm string) int { 238 if len(fm) == 0 || len(to) == 0 { 239 return 0 240 } 241 242 n := len(fm) 243 if len(to) < n { 244 n = len(to) 245 } 246 247 if raceenabled { 248 callerpc := getcallerpc() 249 pc := funcPC(slicestringcopy) 250 racewriterangepc(unsafe.Pointer(&to[0]), uintptr(n), callerpc, pc) 251 } 252 if msanenabled { 253 msanwrite(unsafe.Pointer(&to[0]), uintptr(n)) 254 } 255 256 memmove(unsafe.Pointer(&to[0]), stringStructOf(&fm).str, uintptr(n)) 257 return n 258} 259