1// Copyright 2011 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build aix darwin netbsd openbsd plan9 solaris windows
6
7package runtime
8
9import (
10	"runtime/internal/atomic"
11	"unsafe"
12)
13
14// This implementation depends on OS-specific implementations of
15//
16//	func semacreate(mp *m)
17//		Create a semaphore for mp, if it does not already have one.
18//
19//	func semasleep(ns int64) int32
20//		If ns < 0, acquire m's semaphore and return 0.
21//		If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds.
22//		Return 0 if the semaphore was acquired, -1 if interrupted or timed out.
23//
24//	func semawakeup(mp *m)
25//		Wake up mp, which is or will soon be sleeping on its semaphore.
26//
27const (
28	locked uintptr = 1
29
30	active_spin     = 4
31	active_spin_cnt = 30
32	passive_spin    = 1
33)
34
35func lock(l *mutex) {
36	gp := getg()
37	if gp.m.locks < 0 {
38		throw("runtime·lock: lock count")
39	}
40	gp.m.locks++
41
42	// Speculative grab for lock.
43	if atomic.Casuintptr(&l.key, 0, locked) {
44		return
45	}
46	semacreate(gp.m)
47
48	// On uniprocessor's, no point spinning.
49	// On multiprocessors, spin for ACTIVE_SPIN attempts.
50	spin := 0
51	if ncpu > 1 {
52		spin = active_spin
53	}
54Loop:
55	for i := 0; ; i++ {
56		v := atomic.Loaduintptr(&l.key)
57		if v&locked == 0 {
58			// Unlocked. Try to lock.
59			if atomic.Casuintptr(&l.key, v, v|locked) {
60				return
61			}
62			i = 0
63		}
64		if i < spin {
65			procyield(active_spin_cnt)
66		} else if i < spin+passive_spin {
67			osyield()
68		} else {
69			// Someone else has it.
70			// l->waitm points to a linked list of M's waiting
71			// for this lock, chained through m->nextwaitm.
72			// Queue this M.
73			for {
74				gp.m.nextwaitm = muintptr(v &^ locked)
75				if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) {
76					break
77				}
78				v = atomic.Loaduintptr(&l.key)
79				if v&locked == 0 {
80					continue Loop
81				}
82			}
83			if v&locked != 0 {
84				// Queued. Wait.
85				semasleep(-1)
86				i = 0
87			}
88		}
89	}
90}
91
92//go:nowritebarrier
93// We might not be holding a p in this code.
94func unlock(l *mutex) {
95	gp := getg()
96	var mp *m
97	for {
98		v := atomic.Loaduintptr(&l.key)
99		if v == locked {
100			if atomic.Casuintptr(&l.key, locked, 0) {
101				break
102			}
103		} else {
104			// Other M's are waiting for the lock.
105			// Dequeue an M.
106			mp = muintptr(v &^ locked).ptr()
107			if atomic.Casuintptr(&l.key, v, uintptr(mp.nextwaitm)) {
108				// Dequeued an M.  Wake it.
109				semawakeup(mp)
110				break
111			}
112		}
113	}
114	gp.m.locks--
115	if gp.m.locks < 0 {
116		throw("runtime·unlock: lock count")
117	}
118	if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
119		gp.stackguard0 = stackPreempt
120	}
121}
122
123// One-time notifications.
124func noteclear(n *note) {
125	if GOOS == "aix" {
126		// On AIX, semaphores might not synchronize the memory in some
127		// rare cases. See issue #30189.
128		atomic.Storeuintptr(&n.key, 0)
129	} else {
130		n.key = 0
131	}
132}
133
134func notewakeup(n *note) {
135	var v uintptr
136	for {
137		v = atomic.Loaduintptr(&n.key)
138		if atomic.Casuintptr(&n.key, v, locked) {
139			break
140		}
141	}
142
143	// Successfully set waitm to locked.
144	// What was it before?
145	switch {
146	case v == 0:
147		// Nothing was waiting. Done.
148	case v == locked:
149		// Two notewakeups! Not allowed.
150		throw("notewakeup - double wakeup")
151	default:
152		// Must be the waiting m. Wake it up.
153		semawakeup((*m)(unsafe.Pointer(v)))
154	}
155}
156
157func notesleep(n *note) {
158	gp := getg()
159	if gp != gp.m.g0 {
160		throw("notesleep not on g0")
161	}
162	semacreate(gp.m)
163	if !atomic.Casuintptr(&n.key, 0, uintptr(unsafe.Pointer(gp.m))) {
164		// Must be locked (got wakeup).
165		if n.key != locked {
166			throw("notesleep - waitm out of sync")
167		}
168		return
169	}
170	// Queued. Sleep.
171	gp.m.blocked = true
172	if *cgo_yield == nil {
173		semasleep(-1)
174	} else {
175		// Sleep for an arbitrary-but-moderate interval to poll libc interceptors.
176		const ns = 10e6
177		for atomic.Loaduintptr(&n.key) == 0 {
178			semasleep(ns)
179			asmcgocall(*cgo_yield, nil)
180		}
181	}
182	gp.m.blocked = false
183}
184
185//go:nosplit
186func notetsleep_internal(n *note, ns int64, gp *g, deadline int64) bool {
187	// gp and deadline are logically local variables, but they are written
188	// as parameters so that the stack space they require is charged
189	// to the caller.
190	// This reduces the nosplit footprint of notetsleep_internal.
191	gp = getg()
192
193	// Register for wakeup on n->waitm.
194	if !atomic.Casuintptr(&n.key, 0, uintptr(unsafe.Pointer(gp.m))) {
195		// Must be locked (got wakeup).
196		if n.key != locked {
197			throw("notetsleep - waitm out of sync")
198		}
199		return true
200	}
201	if ns < 0 {
202		// Queued. Sleep.
203		gp.m.blocked = true
204		if *cgo_yield == nil {
205			semasleep(-1)
206		} else {
207			// Sleep in arbitrary-but-moderate intervals to poll libc interceptors.
208			const ns = 10e6
209			for semasleep(ns) < 0 {
210				asmcgocall(*cgo_yield, nil)
211			}
212		}
213		gp.m.blocked = false
214		return true
215	}
216
217	deadline = nanotime() + ns
218	for {
219		// Registered. Sleep.
220		gp.m.blocked = true
221		if *cgo_yield != nil && ns > 10e6 {
222			ns = 10e6
223		}
224		if semasleep(ns) >= 0 {
225			gp.m.blocked = false
226			// Acquired semaphore, semawakeup unregistered us.
227			// Done.
228			return true
229		}
230		if *cgo_yield != nil {
231			asmcgocall(*cgo_yield, nil)
232		}
233		gp.m.blocked = false
234		// Interrupted or timed out. Still registered. Semaphore not acquired.
235		ns = deadline - nanotime()
236		if ns <= 0 {
237			break
238		}
239		// Deadline hasn't arrived. Keep sleeping.
240	}
241
242	// Deadline arrived. Still registered. Semaphore not acquired.
243	// Want to give up and return, but have to unregister first,
244	// so that any notewakeup racing with the return does not
245	// try to grant us the semaphore when we don't expect it.
246	for {
247		v := atomic.Loaduintptr(&n.key)
248		switch v {
249		case uintptr(unsafe.Pointer(gp.m)):
250			// No wakeup yet; unregister if possible.
251			if atomic.Casuintptr(&n.key, v, 0) {
252				return false
253			}
254		case locked:
255			// Wakeup happened so semaphore is available.
256			// Grab it to avoid getting out of sync.
257			gp.m.blocked = true
258			if semasleep(-1) < 0 {
259				throw("runtime: unable to acquire - semaphore out of sync")
260			}
261			gp.m.blocked = false
262			return true
263		default:
264			throw("runtime: unexpected waitm - semaphore out of sync")
265		}
266	}
267}
268
269func notetsleep(n *note, ns int64) bool {
270	gp := getg()
271	if gp != gp.m.g0 {
272		throw("notetsleep not on g0")
273	}
274	semacreate(gp.m)
275	return notetsleep_internal(n, ns, nil, 0)
276}
277
278// same as runtime·notetsleep, but called on user g (not g0)
279// calls only nosplit functions between entersyscallblock/exitsyscall
280func notetsleepg(n *note, ns int64) bool {
281	gp := getg()
282	if gp == gp.m.g0 {
283		throw("notetsleepg on g0")
284	}
285	semacreate(gp.m)
286	entersyscallblock()
287	ok := notetsleep_internal(n, ns, nil, 0)
288	exitsyscall()
289	return ok
290}
291
292func beforeIdle(int64) bool {
293	return false
294}
295
296func checkTimeouts() {}
297