1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package runtime
6
7import (
8	"runtime/internal/atomic"
9	"unsafe"
10)
11
12// These functions cannot have go:noescape annotations,
13// because while ptr does not escape, new does.
14// If new is marked as not escaping, the compiler will make incorrect
15// escape analysis decisions about the pointer value being stored.
16
17// atomicwb performs a write barrier before an atomic pointer write.
18// The caller should guard the call with "if writeBarrier.enabled".
19//
20//go:nosplit
21func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
22	slot := (*uintptr)(unsafe.Pointer(ptr))
23	if !getg().m.p.ptr().wbBuf.putFast(*slot, uintptr(new)) {
24		wbBufFlush(slot, uintptr(new))
25	}
26}
27
28// atomicstorep performs *ptr = new atomically and invokes a write barrier.
29//
30//go:nosplit
31func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
32	if writeBarrier.enabled {
33		atomicwb((*unsafe.Pointer)(ptr), new)
34	}
35	atomic.StorepNoWB(noescape(ptr), new)
36}
37
38// Like above, but implement in terms of sync/atomic's uintptr operations.
39// We cannot just call the runtime routines, because the race detector expects
40// to be able to intercept the sync/atomic forms but not the runtime forms.
41
42//go:linkname sync_atomic_StoreUintptr sync..z2fatomic.StoreUintptr
43func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
44
45//go:linkname sync_atomic_StorePointer sync..z2fatomic.StorePointer
46//go:nosplit
47func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
48	if writeBarrier.enabled {
49		atomicwb(ptr, new)
50	}
51	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
52}
53
54//go:linkname sync_atomic_SwapUintptr sync..z2fatomic.SwapUintptr
55func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
56
57//go:linkname sync_atomic_SwapPointer sync..z2fatomic.SwapPointer
58//go:nosplit
59func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
60	if writeBarrier.enabled {
61		atomicwb(ptr, new)
62	}
63	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
64	return old
65}
66
67//go:linkname sync_atomic_CompareAndSwapUintptr sync..z2fatomic.CompareAndSwapUintptr
68func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
69
70//go:linkname sync_atomic_CompareAndSwapPointer sync..z2fatomic.CompareAndSwapPointer
71//go:nosplit
72func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
73	if writeBarrier.enabled {
74		atomicwb(ptr, new)
75	}
76	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
77}
78