1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Code to check that pointer writes follow the cgo rules. 6// These functions are invoked via the write barrier when debug.cgocheck > 1. 7 8package runtime 9 10import ( 11 "runtime/internal/sys" 12 "unsafe" 13) 14 15const cgoWriteBarrierFail = "Go pointer stored into non-Go memory" 16 17// cgoCheckWriteBarrier is called whenever a pointer is stored into memory. 18// It throws if the program is storing a Go pointer into non-Go memory. 19// 20// This is called from the write barrier, so its entire call tree must 21// be nosplit. 22// 23//go:nosplit 24//go:nowritebarrier 25func cgoCheckWriteBarrier(dst *uintptr, src uintptr) { 26 if !cgoIsGoPointer(unsafe.Pointer(src)) { 27 return 28 } 29 if cgoIsGoPointer(unsafe.Pointer(dst)) { 30 return 31 } 32 33 // If we are running on the system stack then dst might be an 34 // address on the stack, which is OK. 35 g := getg() 36 if g == g.m.g0 || g == g.m.gsignal { 37 return 38 } 39 40 // Allocating memory can write to various mfixalloc structs 41 // that look like they are non-Go memory. 42 if g.m.mallocing != 0 { 43 return 44 } 45 46 // It's OK if writing to memory allocated by persistentalloc. 47 // Do this check last because it is more expensive and rarely true. 48 // If it is false the expense doesn't matter since we are crashing. 49 if inPersistentAlloc(uintptr(unsafe.Pointer(dst))) { 50 return 51 } 52 53 systemstack(func() { 54 println("write of Go pointer", hex(src), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst)))) 55 throw(cgoWriteBarrierFail) 56 }) 57} 58 59// cgoCheckMemmove is called when moving a block of memory. 60// dst and src point off bytes into the value to copy. 61// size is the number of bytes to copy. 62// It throws if the program is copying a block that contains a Go pointer 63// into non-Go memory. 64//go:nosplit 65//go:nowritebarrier 66func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer, off, size uintptr) { 67 if typ.ptrdata == 0 { 68 return 69 } 70 if !cgoIsGoPointer(src) { 71 return 72 } 73 if cgoIsGoPointer(dst) { 74 return 75 } 76 cgoCheckTypedBlock(typ, src, off, size) 77} 78 79// cgoCheckSliceCopy is called when copying n elements of a slice from 80// src to dst. typ is the element type of the slice. 81// It throws if the program is copying slice elements that contain Go pointers 82// into non-Go memory. 83//go:nosplit 84//go:nowritebarrier 85func cgoCheckSliceCopy(typ *_type, dst, src slice, n int) { 86 if typ.ptrdata == 0 { 87 return 88 } 89 if !cgoIsGoPointer(src.array) { 90 return 91 } 92 if cgoIsGoPointer(dst.array) { 93 return 94 } 95 p := src.array 96 for i := 0; i < n; i++ { 97 cgoCheckTypedBlock(typ, p, 0, typ.size) 98 p = add(p, typ.size) 99 } 100} 101 102// cgoCheckTypedBlock checks the block of memory at src, for up to size bytes, 103// and throws if it finds a Go pointer. The type of the memory is typ, 104// and src is off bytes into that type. 105//go:nosplit 106//go:nowritebarrier 107func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { 108 // Anything past typ.ptrdata is not a pointer. 109 if typ.ptrdata <= off { 110 return 111 } 112 if ptrdataSize := typ.ptrdata - off; size > ptrdataSize { 113 size = ptrdataSize 114 } 115 116 if typ.kind&kindGCProg == 0 { 117 cgoCheckBits(src, typ.gcdata, off, size) 118 return 119 } 120 121 // The type has a GC program. Try to find GC bits somewhere else. 122 for _, datap := range activeModules() { 123 if cgoInRange(src, datap.data, datap.edata) { 124 doff := uintptr(src) - datap.data 125 cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size) 126 return 127 } 128 if cgoInRange(src, datap.bss, datap.ebss) { 129 boff := uintptr(src) - datap.bss 130 cgoCheckBits(add(src, -boff), datap.gcbssmask.bytedata, off+boff, size) 131 return 132 } 133 } 134 135 s := spanOfUnchecked(uintptr(src)) 136 if s.state.get() == mSpanManual { 137 // There are no heap bits for value stored on the stack. 138 // For a channel receive src might be on the stack of some 139 // other goroutine, so we can't unwind the stack even if 140 // we wanted to. 141 // We can't expand the GC program without extra storage 142 // space we can't easily get. 143 // Fortunately we have the type information. 144 systemstack(func() { 145 cgoCheckUsingType(typ, src, off, size) 146 }) 147 return 148 } 149 150 // src must be in the regular heap. 151 152 hbits := heapBitsForAddr(uintptr(src)) 153 for i := uintptr(0); i < off+size; i += sys.PtrSize { 154 bits := hbits.bits() 155 if i >= off && bits&bitPointer != 0 { 156 v := *(*unsafe.Pointer)(add(src, i)) 157 if cgoIsGoPointer(v) { 158 throw(cgoWriteBarrierFail) 159 } 160 } 161 hbits = hbits.next() 162 } 163} 164 165// cgoCheckBits checks the block of memory at src, for up to size 166// bytes, and throws if it finds a Go pointer. The gcbits mark each 167// pointer value. The src pointer is off bytes into the gcbits. 168//go:nosplit 169//go:nowritebarrier 170func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) { 171 skipMask := off / sys.PtrSize / 8 172 skipBytes := skipMask * sys.PtrSize * 8 173 ptrmask := addb(gcbits, skipMask) 174 src = add(src, skipBytes) 175 off -= skipBytes 176 size += off 177 var bits uint32 178 for i := uintptr(0); i < size; i += sys.PtrSize { 179 if i&(sys.PtrSize*8-1) == 0 { 180 bits = uint32(*ptrmask) 181 ptrmask = addb(ptrmask, 1) 182 } else { 183 bits >>= 1 184 } 185 if off > 0 { 186 off -= sys.PtrSize 187 } else { 188 if bits&1 != 0 { 189 v := *(*unsafe.Pointer)(add(src, i)) 190 if cgoIsGoPointer(v) { 191 throw(cgoWriteBarrierFail) 192 } 193 } 194 } 195 } 196} 197 198// cgoCheckUsingType is like cgoCheckTypedBlock, but is a last ditch 199// fall back to look for pointers in src using the type information. 200// We only use this when looking at a value on the stack when the type 201// uses a GC program, because otherwise it's more efficient to use the 202// GC bits. This is called on the system stack. 203//go:nowritebarrier 204//go:systemstack 205func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) { 206 if typ.ptrdata == 0 { 207 return 208 } 209 210 // Anything past typ.ptrdata is not a pointer. 211 if typ.ptrdata <= off { 212 return 213 } 214 if ptrdataSize := typ.ptrdata - off; size > ptrdataSize { 215 size = ptrdataSize 216 } 217 218 if typ.kind&kindGCProg == 0 { 219 cgoCheckBits(src, typ.gcdata, off, size) 220 return 221 } 222 switch typ.kind & kindMask { 223 default: 224 throw("can't happen") 225 case kindArray: 226 at := (*arraytype)(unsafe.Pointer(typ)) 227 for i := uintptr(0); i < at.len; i++ { 228 if off < at.elem.size { 229 cgoCheckUsingType(at.elem, src, off, size) 230 } 231 src = add(src, at.elem.size) 232 skipped := off 233 if skipped > at.elem.size { 234 skipped = at.elem.size 235 } 236 checked := at.elem.size - skipped 237 off -= skipped 238 if size <= checked { 239 return 240 } 241 size -= checked 242 } 243 case kindStruct: 244 st := (*structtype)(unsafe.Pointer(typ)) 245 for _, f := range st.fields { 246 if off < f.typ.size { 247 cgoCheckUsingType(f.typ, src, off, size) 248 } 249 src = add(src, f.typ.size) 250 skipped := off 251 if skipped > f.typ.size { 252 skipped = f.typ.size 253 } 254 checked := f.typ.size - skipped 255 off -= skipped 256 if size <= checked { 257 return 258 } 259 size -= checked 260 } 261 } 262} 263