1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import ( 8 "runtime/internal/sys" 9 "unsafe" 10) 11 12type mOS struct{} 13 14//go:noescape 15func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 16 17// Linux futex. 18// 19// futexsleep(uint32 *addr, uint32 val) 20// futexwakeup(uint32 *addr) 21// 22// Futexsleep atomically checks if *addr == val and if so, sleeps on addr. 23// Futexwakeup wakes up threads sleeping on addr. 24// Futexsleep is allowed to wake up spuriously. 25 26const ( 27 _FUTEX_PRIVATE_FLAG = 128 28 _FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG 29 _FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG 30) 31 32// Atomically, 33// if(*addr == val) sleep 34// Might be woken up spuriously; that's allowed. 35// Don't sleep longer than ns; ns < 0 means forever. 36//go:nosplit 37func futexsleep(addr *uint32, val uint32, ns int64) { 38 // Some Linux kernels have a bug where futex of 39 // FUTEX_WAIT returns an internal error code 40 // as an errno. Libpthread ignores the return value 41 // here, and so can we: as it says a few lines up, 42 // spurious wakeups are allowed. 43 if ns < 0 { 44 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0) 45 return 46 } 47 48 var ts timespec 49 ts.setNsec(ns) 50 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0) 51} 52 53// If any procs are sleeping on addr, wake up at most cnt. 54//go:nosplit 55func futexwakeup(addr *uint32, cnt uint32) { 56 ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0) 57 if ret >= 0 { 58 return 59 } 60 61 // I don't know that futex wakeup can return 62 // EAGAIN or EINTR, but if it does, it would be 63 // safe to loop and call futex again. 64 systemstack(func() { 65 print("futexwakeup addr=", addr, " returned ", ret, "\n") 66 }) 67 68 *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 69} 70 71func getproccount() int32 { 72 // This buffer is huge (8 kB) but we are on the system stack 73 // and there should be plenty of space (64 kB). 74 // Also this is a leaf, so we're not holding up the memory for long. 75 // See golang.org/issue/11823. 76 // The suggested behavior here is to keep trying with ever-larger 77 // buffers, but we don't have a dynamic memory allocator at the 78 // moment, so that's a bit tricky and seems like overkill. 79 const maxCPUs = 64 * 1024 80 var buf [maxCPUs / 8]byte 81 r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) 82 if r < 0 { 83 return 1 84 } 85 n := int32(0) 86 for _, v := range buf[:r] { 87 for v != 0 { 88 n += int32(v & 1) 89 v >>= 1 90 } 91 } 92 if n == 0 { 93 n = 1 94 } 95 return n 96} 97 98// Clone, the Linux rfork. 99const ( 100 _CLONE_VM = 0x100 101 _CLONE_FS = 0x200 102 _CLONE_FILES = 0x400 103 _CLONE_SIGHAND = 0x800 104 _CLONE_PTRACE = 0x2000 105 _CLONE_VFORK = 0x4000 106 _CLONE_PARENT = 0x8000 107 _CLONE_THREAD = 0x10000 108 _CLONE_NEWNS = 0x20000 109 _CLONE_SYSVSEM = 0x40000 110 _CLONE_SETTLS = 0x80000 111 _CLONE_PARENT_SETTID = 0x100000 112 _CLONE_CHILD_CLEARTID = 0x200000 113 _CLONE_UNTRACED = 0x800000 114 _CLONE_CHILD_SETTID = 0x1000000 115 _CLONE_STOPPED = 0x2000000 116 _CLONE_NEWUTS = 0x4000000 117 _CLONE_NEWIPC = 0x8000000 118 119 // As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these 120 // flags to be set when creating a thread; attempts to share the other 121 // five but leave SYSVSEM unshared will fail with -EINVAL. 122 // 123 // In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not 124 // use System V semaphores. 125 126 cloneFlags = _CLONE_VM | /* share memory */ 127 _CLONE_FS | /* share cwd, etc */ 128 _CLONE_FILES | /* share fd table */ 129 _CLONE_SIGHAND | /* share sig handler table */ 130 _CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */ 131 _CLONE_THREAD /* revisit - okay for now */ 132) 133 134//go:noescape 135func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32 136 137// May run with m.p==nil, so write barriers are not allowed. 138//go:nowritebarrier 139func newosproc(mp *m) { 140 stk := unsafe.Pointer(mp.g0.stack.hi) 141 /* 142 * note: strace gets confused if we use CLONE_PTRACE here. 143 */ 144 if false { 145 print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", funcPC(clone), " id=", mp.id, " ostk=", &mp, "\n") 146 } 147 148 // Disable signals during clone, so that the new thread starts 149 // with signals disabled. It will enable them in minit. 150 var oset sigset 151 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 152 ret := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(funcPC(mstart))) 153 sigprocmask(_SIG_SETMASK, &oset, nil) 154 155 if ret < 0 { 156 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") 157 if ret == -_EAGAIN { 158 println("runtime: may need to increase max user processes (ulimit -u)") 159 } 160 throw("newosproc") 161 } 162} 163 164// Version of newosproc that doesn't require a valid G. 165//go:nosplit 166func newosproc0(stacksize uintptr, fn unsafe.Pointer) { 167 stack := sysAlloc(stacksize, &memstats.stacks_sys) 168 if stack == nil { 169 write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack))) 170 exit(1) 171 } 172 ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn) 173 if ret < 0 { 174 write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate))) 175 exit(1) 176 } 177} 178 179var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n") 180var failthreadcreate = []byte("runtime: failed to create new OS thread\n") 181 182const ( 183 _AT_NULL = 0 // End of vector 184 _AT_PAGESZ = 6 // System physical page size 185 _AT_HWCAP = 16 // hardware capability bit vector 186 _AT_RANDOM = 25 // introduced in 2.6.29 187 _AT_HWCAP2 = 26 // hardware capability bit vector 2 188) 189 190var procAuxv = []byte("/proc/self/auxv\x00") 191 192var addrspace_vec [1]byte 193 194func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 195 196func sysargs(argc int32, argv **byte) { 197 n := argc + 1 198 199 // skip over argv, envp to get to auxv 200 for argv_index(argv, n) != nil { 201 n++ 202 } 203 204 // skip NULL separator 205 n++ 206 207 // now argv+n is auxv 208 auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*sys.PtrSize)) 209 if sysauxv(auxv[:]) != 0 { 210 return 211 } 212 // In some situations we don't get a loader-provided 213 // auxv, such as when loaded as a library on Android. 214 // Fall back to /proc/self/auxv. 215 fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0) 216 if fd < 0 { 217 // On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to 218 // try using mincore to detect the physical page size. 219 // mincore should return EINVAL when address is not a multiple of system page size. 220 const size = 256 << 10 // size of memory region to allocate 221 p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) 222 if err != 0 { 223 return 224 } 225 var n uintptr 226 for n = 4 << 10; n < size; n <<= 1 { 227 err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0]) 228 if err == 0 { 229 physPageSize = n 230 break 231 } 232 } 233 if physPageSize == 0 { 234 physPageSize = size 235 } 236 munmap(p, size) 237 return 238 } 239 var buf [128]uintptr 240 n = read(fd, noescape(unsafe.Pointer(&buf[0])), int32(unsafe.Sizeof(buf))) 241 closefd(fd) 242 if n < 0 { 243 return 244 } 245 // Make sure buf is terminated, even if we didn't read 246 // the whole file. 247 buf[len(buf)-2] = _AT_NULL 248 sysauxv(buf[:]) 249} 250 251func sysauxv(auxv []uintptr) int { 252 var i int 253 for ; auxv[i] != _AT_NULL; i += 2 { 254 tag, val := auxv[i], auxv[i+1] 255 switch tag { 256 case _AT_RANDOM: 257 // The kernel provides a pointer to 16-bytes 258 // worth of random data. 259 startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:] 260 261 case _AT_PAGESZ: 262 physPageSize = val 263 } 264 265 archauxv(tag, val) 266 vdsoauxv(tag, val) 267 } 268 return i / 2 269} 270 271var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00") 272 273func getHugePageSize() uintptr { 274 var numbuf [20]byte 275 fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0) 276 if fd < 0 { 277 return 0 278 } 279 n := read(fd, noescape(unsafe.Pointer(&numbuf[0])), int32(len(numbuf))) 280 closefd(fd) 281 if n <= 0 { 282 return 0 283 } 284 l := n - 1 // remove trailing newline 285 v, ok := atoi(slicebytetostringtmp(numbuf[:l])) 286 if !ok || v < 0 { 287 v = 0 288 } 289 if v&(v-1) != 0 { 290 // v is not a power of 2 291 return 0 292 } 293 return uintptr(v) 294} 295 296func osinit() { 297 ncpu = getproccount() 298 physHugePageSize = getHugePageSize() 299 osArchInit() 300} 301 302var urandom_dev = []byte("/dev/urandom\x00") 303 304func getRandomData(r []byte) { 305 if startupRandomData != nil { 306 n := copy(r, startupRandomData) 307 extendRandom(r, n) 308 return 309 } 310 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 311 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 312 closefd(fd) 313 extendRandom(r, int(n)) 314} 315 316func goenvs() { 317 goenvs_unix() 318} 319 320// Called to do synchronous initialization of Go code built with 321// -buildmode=c-archive or -buildmode=c-shared. 322// None of the Go runtime is initialized. 323//go:nosplit 324//go:nowritebarrierrec 325func libpreinit() { 326 initsig(true) 327} 328 329// gsignalInitQuirk, if non-nil, is called for every allocated gsignal G. 330// 331// TODO(austin): Remove this after Go 1.15 when we remove the 332// mlockGsignal workaround. 333var gsignalInitQuirk func(gsignal *g) 334 335// Called to initialize a new m (including the bootstrap m). 336// Called on the parent thread (main thread in case of bootstrap), can allocate memory. 337func mpreinit(mp *m) { 338 mp.gsignal = malg(32 * 1024) // Linux wants >= 2K 339 mp.gsignal.m = mp 340 if gsignalInitQuirk != nil { 341 gsignalInitQuirk(mp.gsignal) 342 } 343} 344 345func gettid() uint32 346 347// Called to initialize a new m (including the bootstrap m). 348// Called on the new thread, cannot allocate memory. 349func minit() { 350 minitSignals() 351 352 // Cgo-created threads and the bootstrap m are missing a 353 // procid. We need this for asynchronous preemption and it's 354 // useful in debuggers. 355 getg().m.procid = uint64(gettid()) 356} 357 358// Called from dropm to undo the effect of an minit. 359//go:nosplit 360func unminit() { 361 unminitSignals() 362} 363 364//#ifdef GOARCH_386 365//#define sa_handler k_sa_handler 366//#endif 367 368func sigreturn() 369func sigtramp(sig uint32, info *siginfo, ctx unsafe.Pointer) 370func cgoSigtramp() 371 372//go:noescape 373func sigaltstack(new, old *stackt) 374 375//go:noescape 376func setitimer(mode int32, new, old *itimerval) 377 378//go:noescape 379func rtsigprocmask(how int32, new, old *sigset, size int32) 380 381//go:nosplit 382//go:nowritebarrierrec 383func sigprocmask(how int32, new, old *sigset) { 384 rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new))) 385} 386 387func raise(sig uint32) 388func raiseproc(sig uint32) 389 390//go:noescape 391func sched_getaffinity(pid, len uintptr, buf *byte) int32 392func osyield() 393 394func pipe() (r, w int32, errno int32) 395func pipe2(flags int32) (r, w int32, errno int32) 396func setNonblock(fd int32) 397 398//go:nosplit 399//go:nowritebarrierrec 400func setsig(i uint32, fn uintptr) { 401 var sa sigactiont 402 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART 403 sigfillset(&sa.sa_mask) 404 // Although Linux manpage says "sa_restorer element is obsolete and 405 // should not be used". x86_64 kernel requires it. Only use it on 406 // x86. 407 if GOARCH == "386" || GOARCH == "amd64" { 408 sa.sa_restorer = funcPC(sigreturn) 409 } 410 if fn == funcPC(sighandler) { 411 if iscgo { 412 fn = funcPC(cgoSigtramp) 413 } else { 414 fn = funcPC(sigtramp) 415 } 416 } 417 sa.sa_handler = fn 418 sigaction(i, &sa, nil) 419} 420 421//go:nosplit 422//go:nowritebarrierrec 423func setsigstack(i uint32) { 424 var sa sigactiont 425 sigaction(i, nil, &sa) 426 if sa.sa_flags&_SA_ONSTACK != 0 { 427 return 428 } 429 sa.sa_flags |= _SA_ONSTACK 430 sigaction(i, &sa, nil) 431} 432 433//go:nosplit 434//go:nowritebarrierrec 435func getsig(i uint32) uintptr { 436 var sa sigactiont 437 sigaction(i, nil, &sa) 438 return sa.sa_handler 439} 440 441// setSignaltstackSP sets the ss_sp field of a stackt. 442//go:nosplit 443func setSignalstackSP(s *stackt, sp uintptr) { 444 *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp 445} 446 447//go:nosplit 448func (c *sigctxt) fixsigcode(sig uint32) { 449} 450 451// sysSigaction calls the rt_sigaction system call. 452//go:nosplit 453func sysSigaction(sig uint32, new, old *sigactiont) { 454 if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 { 455 // Workaround for bugs in QEMU user mode emulation. 456 // 457 // QEMU turns calls to the sigaction system call into 458 // calls to the C library sigaction call; the C 459 // library call rejects attempts to call sigaction for 460 // SIGCANCEL (32) or SIGSETXID (33). 461 // 462 // QEMU rejects calling sigaction on SIGRTMAX (64). 463 // 464 // Just ignore the error in these case. There isn't 465 // anything we can do about it anyhow. 466 if sig != 32 && sig != 33 && sig != 64 { 467 // Use system stack to avoid split stack overflow on ppc64/ppc64le. 468 systemstack(func() { 469 throw("sigaction failed") 470 }) 471 } 472 } 473} 474 475// rt_sigaction is implemented in assembly. 476//go:noescape 477func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 478 479func getpid() int 480func tgkill(tgid, tid, sig int) 481 482// signalM sends a signal to mp. 483func signalM(mp *m, sig int) { 484 tgkill(getpid(), int(mp.procid), sig) 485} 486