1// Copyright 2013 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package net
6
7import "sync/atomic"
8
9// fdMutex is a specialized synchronization primitive
10// that manages lifetime of an fd and serializes access
11// to Read and Write methods on netFD.
12type fdMutex struct {
13	state uint64
14	rsema uint32
15	wsema uint32
16}
17
18// fdMutex.state is organized as follows:
19// 1 bit - whether netFD is closed, if set all subsequent lock operations will fail.
20// 1 bit - lock for read operations.
21// 1 bit - lock for write operations.
22// 20 bits - total number of references (read+write+misc).
23// 20 bits - number of outstanding read waiters.
24// 20 bits - number of outstanding write waiters.
25const (
26	mutexClosed  = 1 << 0
27	mutexRLock   = 1 << 1
28	mutexWLock   = 1 << 2
29	mutexRef     = 1 << 3
30	mutexRefMask = (1<<20 - 1) << 3
31	mutexRWait   = 1 << 23
32	mutexRMask   = (1<<20 - 1) << 23
33	mutexWWait   = 1 << 43
34	mutexWMask   = (1<<20 - 1) << 43
35)
36
37// Read operations must do RWLock(true)/RWUnlock(true).
38// Write operations must do RWLock(false)/RWUnlock(false).
39// Misc operations must do Incref/Decref. Misc operations include functions like
40// setsockopt and setDeadline. They need to use Incref/Decref to ensure that
41// they operate on the correct fd in presence of a concurrent Close call
42// (otherwise fd can be closed under their feet).
43// Close operation must do IncrefAndClose/Decref.
44
45// RWLock/Incref return whether fd is open.
46// RWUnlock/Decref return whether fd is closed and there are no remaining references.
47
48func (mu *fdMutex) Incref() bool {
49	for {
50		old := atomic.LoadUint64(&mu.state)
51		if old&mutexClosed != 0 {
52			return false
53		}
54		new := old + mutexRef
55		if new&mutexRefMask == 0 {
56			panic("net: inconsistent fdMutex")
57		}
58		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
59			return true
60		}
61	}
62}
63
64func (mu *fdMutex) IncrefAndClose() bool {
65	for {
66		old := atomic.LoadUint64(&mu.state)
67		if old&mutexClosed != 0 {
68			return false
69		}
70		// Mark as closed and acquire a reference.
71		new := (old | mutexClosed) + mutexRef
72		if new&mutexRefMask == 0 {
73			panic("net: inconsistent fdMutex")
74		}
75		// Remove all read and write waiters.
76		new &^= mutexRMask | mutexWMask
77		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
78			// Wake all read and write waiters,
79			// they will observe closed flag after wakeup.
80			for old&mutexRMask != 0 {
81				old -= mutexRWait
82				runtime_Semrelease(&mu.rsema)
83			}
84			for old&mutexWMask != 0 {
85				old -= mutexWWait
86				runtime_Semrelease(&mu.wsema)
87			}
88			return true
89		}
90	}
91}
92
93func (mu *fdMutex) Decref() bool {
94	for {
95		old := atomic.LoadUint64(&mu.state)
96		if old&mutexRefMask == 0 {
97			panic("net: inconsistent fdMutex")
98		}
99		new := old - mutexRef
100		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
101			return new&(mutexClosed|mutexRefMask) == mutexClosed
102		}
103	}
104}
105
106func (mu *fdMutex) RWLock(read bool) bool {
107	var mutexBit, mutexWait, mutexMask uint64
108	var mutexSema *uint32
109	if read {
110		mutexBit = mutexRLock
111		mutexWait = mutexRWait
112		mutexMask = mutexRMask
113		mutexSema = &mu.rsema
114	} else {
115		mutexBit = mutexWLock
116		mutexWait = mutexWWait
117		mutexMask = mutexWMask
118		mutexSema = &mu.wsema
119	}
120	for {
121		old := atomic.LoadUint64(&mu.state)
122		if old&mutexClosed != 0 {
123			return false
124		}
125		var new uint64
126		if old&mutexBit == 0 {
127			// Lock is free, acquire it.
128			new = (old | mutexBit) + mutexRef
129			if new&mutexRefMask == 0 {
130				panic("net: inconsistent fdMutex")
131			}
132		} else {
133			// Wait for lock.
134			new = old + mutexWait
135			if new&mutexMask == 0 {
136				panic("net: inconsistent fdMutex")
137			}
138		}
139		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
140			if old&mutexBit == 0 {
141				return true
142			}
143			runtime_Semacquire(mutexSema)
144			// The signaller has subtracted mutexWait.
145		}
146	}
147}
148
149func (mu *fdMutex) RWUnlock(read bool) bool {
150	var mutexBit, mutexWait, mutexMask uint64
151	var mutexSema *uint32
152	if read {
153		mutexBit = mutexRLock
154		mutexWait = mutexRWait
155		mutexMask = mutexRMask
156		mutexSema = &mu.rsema
157	} else {
158		mutexBit = mutexWLock
159		mutexWait = mutexWWait
160		mutexMask = mutexWMask
161		mutexSema = &mu.wsema
162	}
163	for {
164		old := atomic.LoadUint64(&mu.state)
165		if old&mutexBit == 0 || old&mutexRefMask == 0 {
166			panic("net: inconsistent fdMutex")
167		}
168		// Drop lock, drop reference and wake read waiter if present.
169		new := (old &^ mutexBit) - mutexRef
170		if old&mutexMask != 0 {
171			new -= mutexWait
172		}
173		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
174			if old&mutexMask != 0 {
175				runtime_Semrelease(mutexSema)
176			}
177			return new&(mutexClosed|mutexRefMask) == mutexClosed
178		}
179	}
180}
181
182// Implemented in runtime package.
183func runtime_Semacquire(sema *uint32)
184func runtime_Semrelease(sema *uint32)
185