1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import "unsafe" 8 9// Call fn with arg as its argument. Return what fn returns. 10// fn is the raw pc value of the entry point of the desired function. 11// Switches to the system stack, if not already there. 12// Preserves the calling point as the location where a profiler traceback will begin. 13//go:nosplit 14func libcCall(fn, arg unsafe.Pointer) int32 { 15 // Leave caller's PC/SP/G around for traceback. 16 gp := getg() 17 var mp *m 18 if gp != nil { 19 mp = gp.m 20 } 21 if mp != nil && mp.libcallsp == 0 { 22 mp.libcallg.set(gp) 23 mp.libcallpc = getcallerpc() 24 // sp must be the last, because once async cpu profiler finds 25 // all three values to be non-zero, it will use them 26 mp.libcallsp = getcallersp() 27 } else { 28 // Make sure we don't reset libcallsp. This makes 29 // libcCall reentrant; We remember the g/pc/sp for the 30 // first call on an M, until that libcCall instance 31 // returns. Reentrance only matters for signals, as 32 // libc never calls back into Go. The tricky case is 33 // where we call libcX from an M and record g/pc/sp. 34 // Before that call returns, a signal arrives on the 35 // same M and the signal handling code calls another 36 // libc function. We don't want that second libcCall 37 // from within the handler to be recorded, and we 38 // don't want that call's completion to zero 39 // libcallsp. 40 // We don't need to set libcall* while we're in a sighandler 41 // (even if we're not currently in libc) because we block all 42 // signals while we're handling a signal. That includes the 43 // profile signal, which is the one that uses the libcall* info. 44 mp = nil 45 } 46 res := asmcgocall(fn, arg) 47 if mp != nil { 48 mp.libcallsp = 0 49 } 50 return res 51} 52 53// The X versions of syscall expect the libc call to return a 64-bit result. 54// Otherwise (the non-X version) expects a 32-bit result. 55// This distinction is required because an error is indicated by returning -1, 56// and we need to know whether to check 32 or 64 bits of the result. 57// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) 58 59//go:linkname syscall_syscall syscall.syscall 60//go:nosplit 61//go:cgo_unsafe_args 62func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 63 entersyscall() 64 libcCall(unsafe.Pointer(funcPC(syscall)), unsafe.Pointer(&fn)) 65 exitsyscall() 66 return 67} 68func syscall() 69 70//go:linkname syscall_syscall6 syscall.syscall6 71//go:nosplit 72//go:cgo_unsafe_args 73func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 74 entersyscall() 75 libcCall(unsafe.Pointer(funcPC(syscall6)), unsafe.Pointer(&fn)) 76 exitsyscall() 77 return 78} 79func syscall6() 80 81//go:linkname syscall_syscall6X syscall.syscall6X 82//go:nosplit 83//go:cgo_unsafe_args 84func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 85 entersyscall() 86 libcCall(unsafe.Pointer(funcPC(syscall6X)), unsafe.Pointer(&fn)) 87 exitsyscall() 88 return 89} 90func syscall6X() 91 92//go:linkname syscall_syscallPtr syscall.syscallPtr 93//go:nosplit 94//go:cgo_unsafe_args 95func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 96 entersyscall() 97 libcCall(unsafe.Pointer(funcPC(syscallPtr)), unsafe.Pointer(&fn)) 98 exitsyscall() 99 return 100} 101func syscallPtr() 102 103//go:linkname syscall_rawSyscall syscall.rawSyscall 104//go:nosplit 105//go:cgo_unsafe_args 106func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 107 libcCall(unsafe.Pointer(funcPC(syscall)), unsafe.Pointer(&fn)) 108 return 109} 110 111//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 112//go:nosplit 113//go:cgo_unsafe_args 114func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 115 libcCall(unsafe.Pointer(funcPC(syscall6)), unsafe.Pointer(&fn)) 116 return 117} 118 119// The *_trampoline functions convert from the Go calling convention to the C calling convention 120// and then call the underlying libc function. They are defined in sys_darwin_$ARCH.s. 121 122//go:nosplit 123//go:cgo_unsafe_args 124func pthread_attr_init(attr *pthreadattr) int32 { 125 return libcCall(unsafe.Pointer(funcPC(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) 126} 127func pthread_attr_init_trampoline() 128 129//go:nosplit 130//go:cgo_unsafe_args 131func pthread_attr_getstacksize(attr *pthreadattr, size *uintptr) int32 { 132 return libcCall(unsafe.Pointer(funcPC(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) 133} 134func pthread_attr_getstacksize_trampoline() 135 136//go:nosplit 137//go:cgo_unsafe_args 138func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { 139 return libcCall(unsafe.Pointer(funcPC(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) 140} 141func pthread_attr_setdetachstate_trampoline() 142 143//go:nosplit 144//go:cgo_unsafe_args 145func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { 146 return libcCall(unsafe.Pointer(funcPC(pthread_create_trampoline)), unsafe.Pointer(&attr)) 147} 148func pthread_create_trampoline() 149 150//go:nosplit 151//go:cgo_unsafe_args 152func raise(sig uint32) { 153 libcCall(unsafe.Pointer(funcPC(raise_trampoline)), unsafe.Pointer(&sig)) 154} 155func raise_trampoline() 156 157//go:nosplit 158//go:cgo_unsafe_args 159func pthread_self() (t pthread) { 160 libcCall(unsafe.Pointer(funcPC(pthread_self_trampoline)), unsafe.Pointer(&t)) 161 return 162} 163func pthread_self_trampoline() 164 165//go:nosplit 166//go:cgo_unsafe_args 167func pthread_kill(t pthread, sig uint32) { 168 libcCall(unsafe.Pointer(funcPC(pthread_kill_trampoline)), unsafe.Pointer(&t)) 169 return 170} 171func pthread_kill_trampoline() 172 173// mmap is used to do low-level memory allocation via mmap. Don't allow stack 174// splits, since this function (used by sysAlloc) is called in a lot of low-level 175// parts of the runtime and callers often assume it won't acquire any locks. 176// go:nosplit 177func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { 178 args := struct { 179 addr unsafe.Pointer 180 n uintptr 181 prot, flags, fd int32 182 off uint32 183 ret1 unsafe.Pointer 184 ret2 int 185 }{addr, n, prot, flags, fd, off, nil, 0} 186 libcCall(unsafe.Pointer(funcPC(mmap_trampoline)), unsafe.Pointer(&args)) 187 return args.ret1, args.ret2 188} 189func mmap_trampoline() 190 191//go:nosplit 192//go:cgo_unsafe_args 193func munmap(addr unsafe.Pointer, n uintptr) { 194 libcCall(unsafe.Pointer(funcPC(munmap_trampoline)), unsafe.Pointer(&addr)) 195} 196func munmap_trampoline() 197 198//go:nosplit 199//go:cgo_unsafe_args 200func madvise(addr unsafe.Pointer, n uintptr, flags int32) { 201 libcCall(unsafe.Pointer(funcPC(madvise_trampoline)), unsafe.Pointer(&addr)) 202} 203func madvise_trampoline() 204 205//go:nosplit 206//go:cgo_unsafe_args 207func read(fd int32, p unsafe.Pointer, n int32) int32 { 208 return libcCall(unsafe.Pointer(funcPC(read_trampoline)), unsafe.Pointer(&fd)) 209} 210func read_trampoline() 211 212func pipe() (r, w int32, errno int32) { 213 var p [2]int32 214 errno = libcCall(unsafe.Pointer(funcPC(pipe_trampoline)), noescape(unsafe.Pointer(&p))) 215 return p[0], p[1], errno 216} 217func pipe_trampoline() 218 219//go:nosplit 220//go:cgo_unsafe_args 221func closefd(fd int32) int32 { 222 return libcCall(unsafe.Pointer(funcPC(close_trampoline)), unsafe.Pointer(&fd)) 223} 224func close_trampoline() 225 226//go:nosplit 227//go:cgo_unsafe_args 228// 229// This is exported via linkname to assembly in runtime/cgo. 230//go:linkname exit 231func exit(code int32) { 232 libcCall(unsafe.Pointer(funcPC(exit_trampoline)), unsafe.Pointer(&code)) 233} 234func exit_trampoline() 235 236//go:nosplit 237//go:cgo_unsafe_args 238func usleep(usec uint32) { 239 libcCall(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) 240} 241func usleep_trampoline() 242 243//go:nosplit 244//go:cgo_unsafe_args 245func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { 246 return libcCall(unsafe.Pointer(funcPC(write_trampoline)), unsafe.Pointer(&fd)) 247} 248func write_trampoline() 249 250//go:nosplit 251//go:cgo_unsafe_args 252func open(name *byte, mode, perm int32) (ret int32) { 253 return libcCall(unsafe.Pointer(funcPC(open_trampoline)), unsafe.Pointer(&name)) 254} 255func open_trampoline() 256 257//go:nosplit 258//go:cgo_unsafe_args 259func nanotime1() int64 { 260 var r struct { 261 t int64 // raw timer 262 numer, denom uint32 // conversion factors. nanoseconds = t * numer / denom. 263 } 264 libcCall(unsafe.Pointer(funcPC(nanotime_trampoline)), unsafe.Pointer(&r)) 265 // Note: Apple seems unconcerned about overflow here. See 266 // https://developer.apple.com/library/content/qa/qa1398/_index.html 267 // Note also, numer == denom == 1 is common. 268 t := r.t 269 if r.numer != 1 { 270 t *= int64(r.numer) 271 } 272 if r.denom != 1 { 273 t /= int64(r.denom) 274 } 275 return t 276} 277func nanotime_trampoline() 278 279//go:nosplit 280//go:cgo_unsafe_args 281func walltime1() (int64, int32) { 282 var t timeval 283 libcCall(unsafe.Pointer(funcPC(walltime_trampoline)), unsafe.Pointer(&t)) 284 return int64(t.tv_sec), 1000 * t.tv_usec 285} 286func walltime_trampoline() 287 288//go:nosplit 289//go:cgo_unsafe_args 290func sigaction(sig uint32, new *usigactiont, old *usigactiont) { 291 libcCall(unsafe.Pointer(funcPC(sigaction_trampoline)), unsafe.Pointer(&sig)) 292} 293func sigaction_trampoline() 294 295//go:nosplit 296//go:cgo_unsafe_args 297func sigprocmask(how uint32, new *sigset, old *sigset) { 298 libcCall(unsafe.Pointer(funcPC(sigprocmask_trampoline)), unsafe.Pointer(&how)) 299} 300func sigprocmask_trampoline() 301 302//go:nosplit 303//go:cgo_unsafe_args 304func sigaltstack(new *stackt, old *stackt) { 305 if new != nil && new.ss_flags&_SS_DISABLE != 0 && new.ss_size == 0 { 306 // Despite the fact that Darwin's sigaltstack man page says it ignores the size 307 // when SS_DISABLE is set, it doesn't. sigaltstack returns ENOMEM 308 // if we don't give it a reasonable size. 309 // ref: http://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20140421/214296.html 310 new.ss_size = 32768 311 } 312 libcCall(unsafe.Pointer(funcPC(sigaltstack_trampoline)), unsafe.Pointer(&new)) 313} 314func sigaltstack_trampoline() 315 316//go:nosplit 317//go:cgo_unsafe_args 318func raiseproc(sig uint32) { 319 libcCall(unsafe.Pointer(funcPC(raiseproc_trampoline)), unsafe.Pointer(&sig)) 320} 321func raiseproc_trampoline() 322 323//go:nosplit 324//go:cgo_unsafe_args 325func setitimer(mode int32, new, old *itimerval) { 326 libcCall(unsafe.Pointer(funcPC(setitimer_trampoline)), unsafe.Pointer(&mode)) 327} 328func setitimer_trampoline() 329 330//go:nosplit 331//go:cgo_unsafe_args 332func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { 333 return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib)) 334} 335func sysctl_trampoline() 336 337//go:nosplit 338//go:cgo_unsafe_args 339func fcntl(fd, cmd, arg int32) int32 { 340 return libcCall(unsafe.Pointer(funcPC(fcntl_trampoline)), unsafe.Pointer(&fd)) 341} 342func fcntl_trampoline() 343 344//go:nosplit 345//go:cgo_unsafe_args 346func kqueue() int32 { 347 v := libcCall(unsafe.Pointer(funcPC(kqueue_trampoline)), nil) 348 return v 349} 350func kqueue_trampoline() 351 352//go:nosplit 353//go:cgo_unsafe_args 354func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { 355 return libcCall(unsafe.Pointer(funcPC(kevent_trampoline)), unsafe.Pointer(&kq)) 356} 357func kevent_trampoline() 358 359//go:nosplit 360//go:cgo_unsafe_args 361func pthread_mutex_init(m *pthreadmutex, attr *pthreadmutexattr) int32 { 362 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_init_trampoline)), unsafe.Pointer(&m)) 363} 364func pthread_mutex_init_trampoline() 365 366//go:nosplit 367//go:cgo_unsafe_args 368func pthread_mutex_lock(m *pthreadmutex) int32 { 369 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_lock_trampoline)), unsafe.Pointer(&m)) 370} 371func pthread_mutex_lock_trampoline() 372 373//go:nosplit 374//go:cgo_unsafe_args 375func pthread_mutex_unlock(m *pthreadmutex) int32 { 376 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_unlock_trampoline)), unsafe.Pointer(&m)) 377} 378func pthread_mutex_unlock_trampoline() 379 380//go:nosplit 381//go:cgo_unsafe_args 382func pthread_cond_init(c *pthreadcond, attr *pthreadcondattr) int32 { 383 return libcCall(unsafe.Pointer(funcPC(pthread_cond_init_trampoline)), unsafe.Pointer(&c)) 384} 385func pthread_cond_init_trampoline() 386 387//go:nosplit 388//go:cgo_unsafe_args 389func pthread_cond_wait(c *pthreadcond, m *pthreadmutex) int32 { 390 return libcCall(unsafe.Pointer(funcPC(pthread_cond_wait_trampoline)), unsafe.Pointer(&c)) 391} 392func pthread_cond_wait_trampoline() 393 394//go:nosplit 395//go:cgo_unsafe_args 396func pthread_cond_timedwait_relative_np(c *pthreadcond, m *pthreadmutex, t *timespec) int32 { 397 return libcCall(unsafe.Pointer(funcPC(pthread_cond_timedwait_relative_np_trampoline)), unsafe.Pointer(&c)) 398} 399func pthread_cond_timedwait_relative_np_trampoline() 400 401//go:nosplit 402//go:cgo_unsafe_args 403func pthread_cond_signal(c *pthreadcond) int32 { 404 return libcCall(unsafe.Pointer(funcPC(pthread_cond_signal_trampoline)), unsafe.Pointer(&c)) 405} 406func pthread_cond_signal_trampoline() 407 408// Not used on Darwin, but must be defined. 409func exitThread(wait *uint32) { 410} 411 412//go:nosplit 413func closeonexec(fd int32) { 414 fcntl(fd, _F_SETFD, _FD_CLOEXEC) 415} 416 417//go:nosplit 418func setNonblock(fd int32) { 419 flags := fcntl(fd, _F_GETFL, 0) 420 fcntl(fd, _F_SETFL, flags|_O_NONBLOCK) 421} 422 423// Tell the linker that the libc_* functions are to be found 424// in a system library, with the libc_ prefix missing. 425 426//go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "/usr/lib/libSystem.B.dylib" 427//go:cgo_import_dynamic libc_pthread_attr_getstacksize pthread_attr_getstacksize "/usr/lib/libSystem.B.dylib" 428//go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "/usr/lib/libSystem.B.dylib" 429//go:cgo_import_dynamic libc_pthread_create pthread_create "/usr/lib/libSystem.B.dylib" 430//go:cgo_import_dynamic libc_pthread_self pthread_self "/usr/lib/libSystem.B.dylib" 431//go:cgo_import_dynamic libc_pthread_kill pthread_kill "/usr/lib/libSystem.B.dylib" 432//go:cgo_import_dynamic libc_exit exit "/usr/lib/libSystem.B.dylib" 433//go:cgo_import_dynamic libc_raise raise "/usr/lib/libSystem.B.dylib" 434 435//go:cgo_import_dynamic libc_open open "/usr/lib/libSystem.B.dylib" 436//go:cgo_import_dynamic libc_close close "/usr/lib/libSystem.B.dylib" 437//go:cgo_import_dynamic libc_read read "/usr/lib/libSystem.B.dylib" 438//go:cgo_import_dynamic libc_write write "/usr/lib/libSystem.B.dylib" 439//go:cgo_import_dynamic libc_pipe pipe "/usr/lib/libSystem.B.dylib" 440 441//go:cgo_import_dynamic libc_mmap mmap "/usr/lib/libSystem.B.dylib" 442//go:cgo_import_dynamic libc_munmap munmap "/usr/lib/libSystem.B.dylib" 443//go:cgo_import_dynamic libc_madvise madvise "/usr/lib/libSystem.B.dylib" 444//go:cgo_import_dynamic libc_error __error "/usr/lib/libSystem.B.dylib" 445//go:cgo_import_dynamic libc_usleep usleep "/usr/lib/libSystem.B.dylib" 446 447//go:cgo_import_dynamic libc_mach_timebase_info mach_timebase_info "/usr/lib/libSystem.B.dylib" 448//go:cgo_import_dynamic libc_mach_absolute_time mach_absolute_time "/usr/lib/libSystem.B.dylib" 449//go:cgo_import_dynamic libc_gettimeofday gettimeofday "/usr/lib/libSystem.B.dylib" 450//go:cgo_import_dynamic libc_sigaction sigaction "/usr/lib/libSystem.B.dylib" 451//go:cgo_import_dynamic libc_pthread_sigmask pthread_sigmask "/usr/lib/libSystem.B.dylib" 452//go:cgo_import_dynamic libc_sigaltstack sigaltstack "/usr/lib/libSystem.B.dylib" 453//go:cgo_import_dynamic libc_getpid getpid "/usr/lib/libSystem.B.dylib" 454//go:cgo_import_dynamic libc_kill kill "/usr/lib/libSystem.B.dylib" 455//go:cgo_import_dynamic libc_setitimer setitimer "/usr/lib/libSystem.B.dylib" 456//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" 457//go:cgo_import_dynamic libc_fcntl fcntl "/usr/lib/libSystem.B.dylib" 458//go:cgo_import_dynamic libc_kqueue kqueue "/usr/lib/libSystem.B.dylib" 459//go:cgo_import_dynamic libc_kevent kevent "/usr/lib/libSystem.B.dylib" 460 461//go:cgo_import_dynamic libc_pthread_mutex_init pthread_mutex_init "/usr/lib/libSystem.B.dylib" 462//go:cgo_import_dynamic libc_pthread_mutex_lock pthread_mutex_lock "/usr/lib/libSystem.B.dylib" 463//go:cgo_import_dynamic libc_pthread_mutex_unlock pthread_mutex_unlock "/usr/lib/libSystem.B.dylib" 464//go:cgo_import_dynamic libc_pthread_cond_init pthread_cond_init "/usr/lib/libSystem.B.dylib" 465//go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" 466//go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" 467//go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" 468 469// Magic incantation to get libSystem actually dynamically linked. 470// TODO: Why does the code require this? See cmd/link/internal/ld/go.go 471//go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib" 472