1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import ( 8 "runtime/internal/sys" 9 "unsafe" 10) 11 12type mOS struct{} 13 14//go:noescape 15func thr_new(param *thrparam, size int32) int32 16 17//go:noescape 18func sigaltstack(new, old *stackt) 19 20//go:noescape 21func sigprocmask(how int32, new, old *sigset) 22 23//go:noescape 24func setitimer(mode int32, new, old *itimerval) 25 26//go:noescape 27func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 28 29func raiseproc(sig uint32) 30 31func thr_self() thread 32func thr_kill(tid thread, sig int) 33 34//go:noescape 35func sys_umtx_op(addr *uint32, mode int32, val uint32, uaddr1 uintptr, ut *umtx_time) int32 36 37func osyield() 38 39func kqueue() int32 40 41//go:noescape 42func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 43 44func pipe() (r, w int32, errno int32) 45func pipe2(flags int32) (r, w int32, errno int32) 46func closeonexec(fd int32) 47func setNonblock(fd int32) 48 49// From FreeBSD's <sys/sysctl.h> 50const ( 51 _CTL_HW = 6 52 _HW_PAGESIZE = 7 53) 54 55var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} 56 57// Undocumented numbers from FreeBSD's lib/libc/gen/sysctlnametomib.c. 58const ( 59 _CTL_QUERY = 0 60 _CTL_QUERY_MIB = 3 61) 62 63// sysctlnametomib fill mib with dynamically assigned sysctl entries of name, 64// return count of effected mib slots, return 0 on error. 65func sysctlnametomib(name []byte, mib *[_CTL_MAXNAME]uint32) uint32 { 66 oid := [2]uint32{_CTL_QUERY, _CTL_QUERY_MIB} 67 miblen := uintptr(_CTL_MAXNAME) 68 if sysctl(&oid[0], 2, (*byte)(unsafe.Pointer(mib)), &miblen, (*byte)(unsafe.Pointer(&name[0])), (uintptr)(len(name))) < 0 { 69 return 0 70 } 71 miblen /= unsafe.Sizeof(uint32(0)) 72 if miblen <= 0 { 73 return 0 74 } 75 return uint32(miblen) 76} 77 78const ( 79 _CPU_CURRENT_PID = -1 // Current process ID. 80) 81 82//go:noescape 83func cpuset_getaffinity(level int, which int, id int64, size int, mask *byte) int32 84 85//go:systemstack 86func getncpu() int32 { 87 // Use a large buffer for the CPU mask. We're on the system 88 // stack, so this is fine, and we can't allocate memory for a 89 // dynamically-sized buffer at this point. 90 const maxCPUs = 64 * 1024 91 var mask [maxCPUs / 8]byte 92 var mib [_CTL_MAXNAME]uint32 93 94 // According to FreeBSD's /usr/src/sys/kern/kern_cpuset.c, 95 // cpuset_getaffinity return ERANGE when provided buffer size exceed the limits in kernel. 96 // Querying kern.smp.maxcpus to calculate maximum buffer size. 97 // See https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=200802 98 99 // Variable kern.smp.maxcpus introduced at Dec 23 2003, revision 123766, 100 // with dynamically assigned sysctl entries. 101 miblen := sysctlnametomib([]byte("kern.smp.maxcpus"), &mib) 102 if miblen == 0 { 103 return 1 104 } 105 106 // Query kern.smp.maxcpus. 107 dstsize := uintptr(4) 108 maxcpus := uint32(0) 109 if sysctl(&mib[0], miblen, (*byte)(unsafe.Pointer(&maxcpus)), &dstsize, nil, 0) != 0 { 110 return 1 111 } 112 113 maskSize := int(maxcpus+7) / 8 114 if maskSize < sys.PtrSize { 115 maskSize = sys.PtrSize 116 } 117 if maskSize > len(mask) { 118 maskSize = len(mask) 119 } 120 121 if cpuset_getaffinity(_CPU_LEVEL_WHICH, _CPU_WHICH_PID, _CPU_CURRENT_PID, 122 maskSize, (*byte)(unsafe.Pointer(&mask[0]))) != 0 { 123 return 1 124 } 125 n := int32(0) 126 for _, v := range mask[:maskSize] { 127 for v != 0 { 128 n += int32(v & 1) 129 v >>= 1 130 } 131 } 132 if n == 0 { 133 return 1 134 } 135 return n 136} 137 138func getPageSize() uintptr { 139 mib := [2]uint32{_CTL_HW, _HW_PAGESIZE} 140 out := uint32(0) 141 nout := unsafe.Sizeof(out) 142 ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) 143 if ret >= 0 { 144 return uintptr(out) 145 } 146 return 0 147} 148 149// FreeBSD's umtx_op syscall is effectively the same as Linux's futex, and 150// thus the code is largely similar. See Linux implementation 151// and lock_futex.go for comments. 152 153//go:nosplit 154func futexsleep(addr *uint32, val uint32, ns int64) { 155 systemstack(func() { 156 futexsleep1(addr, val, ns) 157 }) 158} 159 160func futexsleep1(addr *uint32, val uint32, ns int64) { 161 var utp *umtx_time 162 if ns >= 0 { 163 var ut umtx_time 164 ut._clockid = _CLOCK_MONOTONIC 165 ut._timeout.setNsec(ns) 166 utp = &ut 167 } 168 ret := sys_umtx_op(addr, _UMTX_OP_WAIT_UINT_PRIVATE, val, unsafe.Sizeof(*utp), utp) 169 if ret >= 0 || ret == -_EINTR { 170 return 171 } 172 print("umtx_wait addr=", addr, " val=", val, " ret=", ret, "\n") 173 *(*int32)(unsafe.Pointer(uintptr(0x1005))) = 0x1005 174} 175 176//go:nosplit 177func futexwakeup(addr *uint32, cnt uint32) { 178 ret := sys_umtx_op(addr, _UMTX_OP_WAKE_PRIVATE, cnt, 0, nil) 179 if ret >= 0 { 180 return 181 } 182 183 systemstack(func() { 184 print("umtx_wake_addr=", addr, " ret=", ret, "\n") 185 }) 186} 187 188func thr_start() 189 190// May run with m.p==nil, so write barriers are not allowed. 191//go:nowritebarrier 192func newosproc(mp *m) { 193 stk := unsafe.Pointer(mp.g0.stack.hi) 194 if false { 195 print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " thr_start=", funcPC(thr_start), " id=", mp.id, " ostk=", &mp, "\n") 196 } 197 198 param := thrparam{ 199 start_func: funcPC(thr_start), 200 arg: unsafe.Pointer(mp), 201 stack_base: mp.g0.stack.lo, 202 stack_size: uintptr(stk) - mp.g0.stack.lo, 203 child_tid: nil, // minit will record tid 204 parent_tid: nil, 205 tls_base: unsafe.Pointer(&mp.tls[0]), 206 tls_size: unsafe.Sizeof(mp.tls), 207 } 208 209 var oset sigset 210 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 211 // TODO: Check for error. 212 ret := thr_new(¶m, int32(unsafe.Sizeof(param))) 213 sigprocmask(_SIG_SETMASK, &oset, nil) 214 if ret < 0 { 215 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") 216 throw("newosproc") 217 } 218} 219 220// Version of newosproc that doesn't require a valid G. 221//go:nosplit 222func newosproc0(stacksize uintptr, fn unsafe.Pointer) { 223 stack := sysAlloc(stacksize, &memstats.stacks_sys) 224 if stack == nil { 225 write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack))) 226 exit(1) 227 } 228 // This code "knows" it's being called once from the library 229 // initialization code, and so it's using the static m0 for the 230 // tls and procid (thread) pointers. thr_new() requires the tls 231 // pointers, though the tid pointers can be nil. 232 // However, newosproc0 is currently unreachable because builds 233 // utilizing c-shared/c-archive force external linking. 234 param := thrparam{ 235 start_func: funcPC(fn), 236 arg: nil, 237 stack_base: uintptr(stack), //+stacksize? 238 stack_size: stacksize, 239 child_tid: nil, // minit will record tid 240 parent_tid: nil, 241 tls_base: unsafe.Pointer(&m0.tls[0]), 242 tls_size: unsafe.Sizeof(m0.tls), 243 } 244 245 var oset sigset 246 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 247 ret := thr_new(¶m, int32(unsafe.Sizeof(param))) 248 sigprocmask(_SIG_SETMASK, &oset, nil) 249 if ret < 0 { 250 write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate))) 251 exit(1) 252 } 253} 254 255var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n") 256var failthreadcreate = []byte("runtime: failed to create new OS thread\n") 257 258// Called to do synchronous initialization of Go code built with 259// -buildmode=c-archive or -buildmode=c-shared. 260// None of the Go runtime is initialized. 261//go:nosplit 262//go:nowritebarrierrec 263func libpreinit() { 264 initsig(true) 265} 266 267func osinit() { 268 ncpu = getncpu() 269 if physPageSize == 0 { 270 physPageSize = getPageSize() 271 } 272} 273 274var urandom_dev = []byte("/dev/urandom\x00") 275 276//go:nosplit 277func getRandomData(r []byte) { 278 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 279 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 280 closefd(fd) 281 extendRandom(r, int(n)) 282} 283 284func goenvs() { 285 goenvs_unix() 286} 287 288// Called to initialize a new m (including the bootstrap m). 289// Called on the parent thread (main thread in case of bootstrap), can allocate memory. 290func mpreinit(mp *m) { 291 mp.gsignal = malg(32 * 1024) 292 mp.gsignal.m = mp 293} 294 295// Called to initialize a new m (including the bootstrap m). 296// Called on the new thread, cannot allocate memory. 297func minit() { 298 getg().m.procid = uint64(thr_self()) 299 300 // On FreeBSD before about April 2017 there was a bug such 301 // that calling execve from a thread other than the main 302 // thread did not reset the signal stack. That would confuse 303 // minitSignals, which calls minitSignalStack, which checks 304 // whether there is currently a signal stack and uses it if 305 // present. To avoid this confusion, explicitly disable the 306 // signal stack on the main thread when not running in a 307 // library. This can be removed when we are confident that all 308 // FreeBSD users are running a patched kernel. See issue #15658. 309 if gp := getg(); !isarchive && !islibrary && gp.m == &m0 && gp == gp.m.g0 { 310 st := stackt{ss_flags: _SS_DISABLE} 311 sigaltstack(&st, nil) 312 } 313 314 minitSignals() 315} 316 317// Called from dropm to undo the effect of an minit. 318//go:nosplit 319func unminit() { 320 unminitSignals() 321} 322 323func sigtramp() 324 325type sigactiont struct { 326 sa_handler uintptr 327 sa_flags int32 328 sa_mask sigset 329} 330 331// See os_freebsd2.go, os_freebsd_amd64.go for setsig function 332 333//go:nosplit 334//go:nowritebarrierrec 335func setsigstack(i uint32) { 336 var sa sigactiont 337 sigaction(i, nil, &sa) 338 if sa.sa_flags&_SA_ONSTACK != 0 { 339 return 340 } 341 sa.sa_flags |= _SA_ONSTACK 342 sigaction(i, &sa, nil) 343} 344 345//go:nosplit 346//go:nowritebarrierrec 347func getsig(i uint32) uintptr { 348 var sa sigactiont 349 sigaction(i, nil, &sa) 350 return sa.sa_handler 351} 352 353// setSignaltstackSP sets the ss_sp field of a stackt. 354//go:nosplit 355func setSignalstackSP(s *stackt, sp uintptr) { 356 s.ss_sp = sp 357} 358 359//go:nosplit 360//go:nowritebarrierrec 361func sigaddset(mask *sigset, i int) { 362 mask.__bits[(i-1)/32] |= 1 << ((uint32(i) - 1) & 31) 363} 364 365func sigdelset(mask *sigset, i int) { 366 mask.__bits[(i-1)/32] &^= 1 << ((uint32(i) - 1) & 31) 367} 368 369//go:nosplit 370func (c *sigctxt) fixsigcode(sig uint32) { 371} 372 373func sysargs(argc int32, argv **byte) { 374 n := argc + 1 375 376 // skip over argv, envp to get to auxv 377 for argv_index(argv, n) != nil { 378 n++ 379 } 380 381 // skip NULL separator 382 n++ 383 384 // now argv+n is auxv 385 auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*sys.PtrSize)) 386 sysauxv(auxv[:]) 387} 388 389const ( 390 _AT_NULL = 0 // Terminates the vector 391 _AT_PAGESZ = 6 // Page size in bytes 392 _AT_TIMEKEEP = 22 // Pointer to timehands. 393 _AT_HWCAP = 25 // CPU feature flags 394 _AT_HWCAP2 = 26 // CPU feature flags 2 395) 396 397func sysauxv(auxv []uintptr) { 398 for i := 0; auxv[i] != _AT_NULL; i += 2 { 399 tag, val := auxv[i], auxv[i+1] 400 switch tag { 401 // _AT_NCPUS from auxv shouldn't be used due to golang.org/issue/15206 402 case _AT_PAGESZ: 403 physPageSize = val 404 case _AT_TIMEKEEP: 405 timekeepSharedPage = (*vdsoTimekeep)(unsafe.Pointer(val)) 406 } 407 408 archauxv(tag, val) 409 } 410} 411 412// sysSigaction calls the sigaction system call. 413//go:nosplit 414func sysSigaction(sig uint32, new, old *sigactiont) { 415 // Use system stack to avoid split stack overflow on amd64 416 if asmSigaction(uintptr(sig), new, old) != 0 { 417 systemstack(func() { 418 throw("sigaction failed") 419 }) 420 } 421} 422 423// asmSigaction is implemented in assembly. 424//go:noescape 425func asmSigaction(sig uintptr, new, old *sigactiont) int32 426 427// raise sends a signal to the calling thread. 428// 429// It must be nosplit because it is used by the signal handler before 430// it definitely has a Go stack. 431// 432//go:nosplit 433func raise(sig uint32) { 434 thr_kill(thr_self(), int(sig)) 435} 436 437func signalM(mp *m, sig int) { 438 thr_kill(thread(mp.procid), sig) 439} 440