1// Copyright 2013 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build linux
6
7package runtime
8
9import "unsafe"
10
11//extern epoll_create
12func epollcreate(size int32) int32
13
14//extern epoll_create1
15func epollcreate1(flags int32) int32
16
17//go:noescape
18//extern epoll_ctl
19func epollctl(epfd, op, fd int32, ev *epollevent) int32
20
21//go:noescape
22//extern epoll_wait
23func epollwait(epfd int32, ev *epollevent, nev, timeout int32) int32
24
25var (
26	epfd int32 = -1 // epoll descriptor
27
28	netpollBreakRd, netpollBreakWr uintptr // for netpollBreak
29)
30
31func netpollinit() {
32	epfd = epollcreate1(_EPOLL_CLOEXEC)
33	if epfd < 0 {
34		epfd = epollcreate(1024)
35		if epfd < 0 {
36			println("runtime: epollcreate failed with", -epfd)
37			throw("runtime: netpollinit failed")
38		}
39		closeonexec(epfd)
40	}
41	r, w, cerrno := nonblockingPipe()
42	if cerrno != 0 {
43		println("runtime: pipe failed with", cerrno)
44		throw("runtime: pipe failed")
45	}
46	ev := epollevent{
47		events: _EPOLLIN,
48	}
49	*(**uintptr)(unsafe.Pointer(&ev.data)) = &netpollBreakRd
50	if epollctl(epfd, _EPOLL_CTL_ADD, r, &ev) < 0 {
51		cerrno = int32(errno())
52	}
53	if cerrno != 0 {
54		println("runtime: epollctl failed with", cerrno)
55		throw("runtime: epollctl failed")
56	}
57	netpollBreakRd = uintptr(r)
58	netpollBreakWr = uintptr(w)
59}
60
61func netpollIsPollDescriptor(fd uintptr) bool {
62	return fd == uintptr(epfd) || fd == netpollBreakRd || fd == netpollBreakWr
63}
64
65func netpollopen(fd uintptr, pd *pollDesc) int32 {
66	var ev epollevent
67	ev.events = _EPOLLIN | _EPOLLOUT | _EPOLLRDHUP | _EPOLLETpos
68	*(**pollDesc)(unsafe.Pointer(&ev.data)) = pd
69	if epollctl(epfd, _EPOLL_CTL_ADD, int32(fd), &ev) < 0 {
70		return int32(errno())
71	}
72	return 0
73}
74
75func netpollclose(fd uintptr) int32 {
76	var ev epollevent
77	if epollctl(epfd, _EPOLL_CTL_DEL, int32(fd), &ev) < 0 {
78		return int32(errno())
79	}
80	return 0
81}
82
83func netpollarm(pd *pollDesc, mode int) {
84	throw("runtime: unused")
85}
86
87// netpollBreak interrupts an epollwait.
88func netpollBreak() {
89	for {
90		var b byte
91		n := write(netpollBreakWr, unsafe.Pointer(&b), 1)
92		if n == 1 {
93			break
94		}
95		if n == -_EINTR {
96			continue
97		}
98		if n == -_EAGAIN {
99			return
100		}
101		println("runtime: netpollBreak write failed with", -n)
102		throw("runtime: netpollBreak write failed")
103	}
104}
105
106// netpoll checks for ready network connections.
107// Returns list of goroutines that become runnable.
108// delay < 0: blocks indefinitely
109// delay == 0: does not block, just polls
110// delay > 0: block for up to that many nanoseconds
111func netpoll(delay int64) gList {
112	if epfd == -1 {
113		return gList{}
114	}
115	var waitms int32
116	if delay < 0 {
117		waitms = -1
118	} else if delay == 0 {
119		waitms = 0
120	} else if delay < 1e6 {
121		waitms = 1
122	} else if delay < 1e15 {
123		waitms = int32(delay / 1e6)
124	} else {
125		// An arbitrary cap on how long to wait for a timer.
126		// 1e9 ms == ~11.5 days.
127		waitms = 1e9
128	}
129	var events [128]epollevent
130retry:
131	n := epollwait(epfd, &events[0], int32(len(events)), waitms)
132	if n < 0 {
133		e := errno()
134		if e != _EINTR {
135			println("runtime: epollwait on fd", epfd, "failed with", e)
136			throw("runtime: netpoll failed")
137		}
138		// If a timed sleep was interrupted, just return to
139		// recalculate how long we should sleep now.
140		if waitms > 0 {
141			return gList{}
142		}
143		goto retry
144	}
145	var toRun gList
146	for i := int32(0); i < n; i++ {
147		ev := &events[i]
148		if ev.events == 0 {
149			continue
150		}
151
152		if *(**uintptr)(unsafe.Pointer(&ev.data)) == &netpollBreakRd {
153			if ev.events != _EPOLLIN {
154				println("runtime: netpoll: break fd ready for", ev.events)
155				throw("runtime: netpoll: break fd ready for something unexpected")
156			}
157			if delay != 0 {
158				// netpollBreak could be picked up by a
159				// nonblocking poll. Only read the byte
160				// if blocking.
161				var tmp [16]byte
162				read(int32(netpollBreakRd), noescape(unsafe.Pointer(&tmp[0])), int32(len(tmp)))
163			}
164			continue
165		}
166
167		var mode int32
168		if ev.events&(_EPOLLIN|_EPOLLRDHUP|_EPOLLHUP|_EPOLLERR) != 0 {
169			mode += 'r'
170		}
171		if ev.events&(_EPOLLOUT|_EPOLLHUP|_EPOLLERR) != 0 {
172			mode += 'w'
173		}
174		if mode != 0 {
175			pd := *(**pollDesc)(unsafe.Pointer(&ev.data))
176			pd.everr = false
177			if ev.events == _EPOLLERR {
178				pd.everr = true
179			}
180			netpollready(&toRun, pd, mode)
181		}
182	}
183	return toRun
184}
185