1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import "unsafe" 8 9// Call fn with arg as its argument. Return what fn returns. 10// fn is the raw pc value of the entry point of the desired function. 11// Switches to the system stack, if not already there. 12// Preserves the calling point as the location where a profiler traceback will begin. 13//go:nosplit 14func libcCall(fn, arg unsafe.Pointer) int32 { 15 // Leave caller's PC/SP/G around for traceback. 16 gp := getg() 17 var mp *m 18 if gp != nil { 19 mp = gp.m 20 } 21 if mp != nil && mp.libcallsp == 0 { 22 mp.libcallg.set(gp) 23 mp.libcallpc = getcallerpc() 24 // sp must be the last, because once async cpu profiler finds 25 // all three values to be non-zero, it will use them 26 mp.libcallsp = getcallersp() 27 } else { 28 // Make sure we don't reset libcallsp. This makes 29 // libcCall reentrant; We remember the g/pc/sp for the 30 // first call on an M, until that libcCall instance 31 // returns. Reentrance only matters for signals, as 32 // libc never calls back into Go. The tricky case is 33 // where we call libcX from an M and record g/pc/sp. 34 // Before that call returns, a signal arrives on the 35 // same M and the signal handling code calls another 36 // libc function. We don't want that second libcCall 37 // from within the handler to be recorded, and we 38 // don't want that call's completion to zero 39 // libcallsp. 40 // We don't need to set libcall* while we're in a sighandler 41 // (even if we're not currently in libc) because we block all 42 // signals while we're handling a signal. That includes the 43 // profile signal, which is the one that uses the libcall* info. 44 mp = nil 45 } 46 res := asmcgocall(fn, arg) 47 if mp != nil { 48 mp.libcallsp = 0 49 } 50 return res 51} 52 53// The X versions of syscall expect the libc call to return a 64-bit result. 54// Otherwise (the non-X version) expects a 32-bit result. 55// This distinction is required because an error is indicated by returning -1, 56// and we need to know whether to check 32 or 64 bits of the result. 57// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) 58 59//go:linkname syscall_syscall syscall.syscall 60//go:nosplit 61//go:cgo_unsafe_args 62func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 63 entersyscallblock() 64 libcCall(unsafe.Pointer(funcPC(syscall)), unsafe.Pointer(&fn)) 65 exitsyscall() 66 return 67} 68func syscall() 69 70//go:linkname syscall_syscall6 syscall.syscall6 71//go:nosplit 72//go:cgo_unsafe_args 73func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 74 entersyscallblock() 75 libcCall(unsafe.Pointer(funcPC(syscall6)), unsafe.Pointer(&fn)) 76 exitsyscall() 77 return 78} 79func syscall6() 80 81//go:linkname syscall_syscall6X syscall.syscall6X 82//go:nosplit 83//go:cgo_unsafe_args 84func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 85 entersyscallblock() 86 libcCall(unsafe.Pointer(funcPC(syscall6X)), unsafe.Pointer(&fn)) 87 exitsyscall() 88 return 89} 90func syscall6X() 91 92//go:linkname syscall_rawSyscall syscall.rawSyscall 93//go:nosplit 94//go:cgo_unsafe_args 95func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 96 libcCall(unsafe.Pointer(funcPC(syscall)), unsafe.Pointer(&fn)) 97 return 98} 99 100//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 101//go:nosplit 102//go:cgo_unsafe_args 103func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 104 libcCall(unsafe.Pointer(funcPC(syscall6)), unsafe.Pointer(&fn)) 105 return 106} 107 108// The *_trampoline functions convert from the Go calling convention to the C calling convention 109// and then call the underlying libc function. They are defined in sys_darwin_$ARCH.s. 110 111//go:nosplit 112//go:cgo_unsafe_args 113func pthread_attr_init(attr *pthreadattr) int32 { 114 return libcCall(unsafe.Pointer(funcPC(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) 115} 116func pthread_attr_init_trampoline() 117 118//go:nosplit 119//go:cgo_unsafe_args 120func pthread_attr_setstacksize(attr *pthreadattr, size uintptr) int32 { 121 return libcCall(unsafe.Pointer(funcPC(pthread_attr_setstacksize_trampoline)), unsafe.Pointer(&attr)) 122} 123func pthread_attr_setstacksize_trampoline() 124 125//go:nosplit 126//go:cgo_unsafe_args 127func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { 128 return libcCall(unsafe.Pointer(funcPC(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) 129} 130func pthread_attr_setdetachstate_trampoline() 131 132//go:nosplit 133//go:cgo_unsafe_args 134func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { 135 return libcCall(unsafe.Pointer(funcPC(pthread_create_trampoline)), unsafe.Pointer(&attr)) 136} 137func pthread_create_trampoline() 138 139//go:nosplit 140//go:cgo_unsafe_args 141func raise(sig uint32) { 142 libcCall(unsafe.Pointer(funcPC(raise_trampoline)), unsafe.Pointer(&sig)) 143} 144func raise_trampoline() 145 146//go:nosplit 147//go:cgo_unsafe_args 148func pthread_self() (t pthread) { 149 libcCall(unsafe.Pointer(funcPC(pthread_self_trampoline)), unsafe.Pointer(&t)) 150 return 151} 152func pthread_self_trampoline() 153 154func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { 155 args := struct { 156 addr unsafe.Pointer 157 n uintptr 158 prot, flags, fd int32 159 off uint32 160 ret1 unsafe.Pointer 161 ret2 int 162 }{addr, n, prot, flags, fd, off, nil, 0} 163 libcCall(unsafe.Pointer(funcPC(mmap_trampoline)), unsafe.Pointer(&args)) 164 return args.ret1, args.ret2 165} 166func mmap_trampoline() 167 168//go:nosplit 169//go:cgo_unsafe_args 170func munmap(addr unsafe.Pointer, n uintptr) { 171 libcCall(unsafe.Pointer(funcPC(munmap_trampoline)), unsafe.Pointer(&addr)) 172} 173func munmap_trampoline() 174 175//go:nosplit 176//go:cgo_unsafe_args 177func madvise(addr unsafe.Pointer, n uintptr, flags int32) { 178 libcCall(unsafe.Pointer(funcPC(madvise_trampoline)), unsafe.Pointer(&addr)) 179} 180func madvise_trampoline() 181 182//go:nosplit 183//go:cgo_unsafe_args 184func read(fd int32, p unsafe.Pointer, n int32) int32 { 185 return libcCall(unsafe.Pointer(funcPC(read_trampoline)), unsafe.Pointer(&fd)) 186} 187func read_trampoline() 188 189//go:nosplit 190//go:cgo_unsafe_args 191func closefd(fd int32) int32 { 192 return libcCall(unsafe.Pointer(funcPC(close_trampoline)), unsafe.Pointer(&fd)) 193} 194func close_trampoline() 195 196//go:nosplit 197//go:cgo_unsafe_args 198func exit(code int32) { 199 libcCall(unsafe.Pointer(funcPC(exit_trampoline)), unsafe.Pointer(&code)) 200} 201func exit_trampoline() 202 203//go:nosplit 204//go:cgo_unsafe_args 205func usleep(usec uint32) { 206 libcCall(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) 207} 208func usleep_trampoline() 209 210//go:nosplit 211//go:cgo_unsafe_args 212func write(fd uintptr, p unsafe.Pointer, n int32) int32 { 213 return libcCall(unsafe.Pointer(funcPC(write_trampoline)), unsafe.Pointer(&fd)) 214} 215func write_trampoline() 216 217//go:nosplit 218//go:cgo_unsafe_args 219func open(name *byte, mode, perm int32) (ret int32) { 220 return libcCall(unsafe.Pointer(funcPC(open_trampoline)), unsafe.Pointer(&name)) 221} 222func open_trampoline() 223 224//go:nosplit 225//go:cgo_unsafe_args 226func nanotime() int64 { 227 var r struct { 228 t int64 // raw timer 229 numer, denom uint32 // conversion factors. nanoseconds = t * numer / denom. 230 } 231 libcCall(unsafe.Pointer(funcPC(nanotime_trampoline)), unsafe.Pointer(&r)) 232 // Note: Apple seems unconcerned about overflow here. See 233 // https://developer.apple.com/library/content/qa/qa1398/_index.html 234 // Note also, numer == denom == 1 is common. 235 t := r.t 236 if r.numer != 1 { 237 t *= int64(r.numer) 238 } 239 if r.denom != 1 { 240 t /= int64(r.denom) 241 } 242 return t 243} 244func nanotime_trampoline() 245 246//go:nosplit 247//go:cgo_unsafe_args 248func walltime() (int64, int32) { 249 var t timeval 250 libcCall(unsafe.Pointer(funcPC(walltime_trampoline)), unsafe.Pointer(&t)) 251 return int64(t.tv_sec), 1000 * t.tv_usec 252} 253func walltime_trampoline() 254 255//go:nosplit 256//go:cgo_unsafe_args 257func sigaction(sig uint32, new *usigactiont, old *usigactiont) { 258 libcCall(unsafe.Pointer(funcPC(sigaction_trampoline)), unsafe.Pointer(&sig)) 259} 260func sigaction_trampoline() 261 262//go:nosplit 263//go:cgo_unsafe_args 264func sigprocmask(how uint32, new *sigset, old *sigset) { 265 libcCall(unsafe.Pointer(funcPC(sigprocmask_trampoline)), unsafe.Pointer(&how)) 266} 267func sigprocmask_trampoline() 268 269//go:nosplit 270//go:cgo_unsafe_args 271func sigaltstack(new *stackt, old *stackt) { 272 if new != nil && new.ss_flags&_SS_DISABLE != 0 && new.ss_size == 0 { 273 // Despite the fact that Darwin's sigaltstack man page says it ignores the size 274 // when SS_DISABLE is set, it doesn't. sigaltstack returns ENOMEM 275 // if we don't give it a reasonable size. 276 // ref: http://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20140421/214296.html 277 new.ss_size = 32768 278 } 279 libcCall(unsafe.Pointer(funcPC(sigaltstack_trampoline)), unsafe.Pointer(&new)) 280} 281func sigaltstack_trampoline() 282 283//go:nosplit 284//go:cgo_unsafe_args 285func raiseproc(sig uint32) { 286 libcCall(unsafe.Pointer(funcPC(raiseproc_trampoline)), unsafe.Pointer(&sig)) 287} 288func raiseproc_trampoline() 289 290//go:nosplit 291//go:cgo_unsafe_args 292func setitimer(mode int32, new, old *itimerval) { 293 libcCall(unsafe.Pointer(funcPC(setitimer_trampoline)), unsafe.Pointer(&mode)) 294} 295func setitimer_trampoline() 296 297//go:nosplit 298//go:cgo_unsafe_args 299func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { 300 return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib)) 301} 302func sysctl_trampoline() 303 304//go:nosplit 305//go:cgo_unsafe_args 306func fcntl(fd, cmd, arg int32) int32 { 307 return libcCall(unsafe.Pointer(funcPC(fcntl_trampoline)), unsafe.Pointer(&fd)) 308} 309func fcntl_trampoline() 310 311//go:nosplit 312//go:cgo_unsafe_args 313func kqueue() int32 { 314 v := libcCall(unsafe.Pointer(funcPC(kqueue_trampoline)), nil) 315 return v 316} 317func kqueue_trampoline() 318 319//go:nosplit 320//go:cgo_unsafe_args 321func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { 322 return libcCall(unsafe.Pointer(funcPC(kevent_trampoline)), unsafe.Pointer(&kq)) 323} 324func kevent_trampoline() 325 326//go:nosplit 327//go:cgo_unsafe_args 328func pthread_mutex_init(m *pthreadmutex, attr *pthreadmutexattr) int32 { 329 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_init_trampoline)), unsafe.Pointer(&m)) 330} 331func pthread_mutex_init_trampoline() 332 333//go:nosplit 334//go:cgo_unsafe_args 335func pthread_mutex_lock(m *pthreadmutex) int32 { 336 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_lock_trampoline)), unsafe.Pointer(&m)) 337} 338func pthread_mutex_lock_trampoline() 339 340//go:nosplit 341//go:cgo_unsafe_args 342func pthread_mutex_unlock(m *pthreadmutex) int32 { 343 return libcCall(unsafe.Pointer(funcPC(pthread_mutex_unlock_trampoline)), unsafe.Pointer(&m)) 344} 345func pthread_mutex_unlock_trampoline() 346 347//go:nosplit 348//go:cgo_unsafe_args 349func pthread_cond_init(c *pthreadcond, attr *pthreadcondattr) int32 { 350 return libcCall(unsafe.Pointer(funcPC(pthread_cond_init_trampoline)), unsafe.Pointer(&c)) 351} 352func pthread_cond_init_trampoline() 353 354//go:nosplit 355//go:cgo_unsafe_args 356func pthread_cond_wait(c *pthreadcond, m *pthreadmutex) int32 { 357 return libcCall(unsafe.Pointer(funcPC(pthread_cond_wait_trampoline)), unsafe.Pointer(&c)) 358} 359func pthread_cond_wait_trampoline() 360 361//go:nosplit 362//go:cgo_unsafe_args 363func pthread_cond_timedwait_relative_np(c *pthreadcond, m *pthreadmutex, t *timespec) int32 { 364 return libcCall(unsafe.Pointer(funcPC(pthread_cond_timedwait_relative_np_trampoline)), unsafe.Pointer(&c)) 365} 366func pthread_cond_timedwait_relative_np_trampoline() 367 368//go:nosplit 369//go:cgo_unsafe_args 370func pthread_cond_signal(c *pthreadcond) int32 { 371 return libcCall(unsafe.Pointer(funcPC(pthread_cond_signal_trampoline)), unsafe.Pointer(&c)) 372} 373func pthread_cond_signal_trampoline() 374 375// Not used on Darwin, but must be defined. 376func exitThread(wait *uint32) { 377} 378 379//go:nosplit 380func closeonexec(fd int32) { 381 fcntl(fd, _F_SETFD, _FD_CLOEXEC) 382} 383 384// Tell the linker that the libc_* functions are to be found 385// in a system library, with the libc_ prefix missing. 386 387//go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "/usr/lib/libSystem.B.dylib" 388//go:cgo_import_dynamic libc_pthread_attr_setstacksize pthread_attr_setstacksize "/usr/lib/libSystem.B.dylib" 389//go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "/usr/lib/libSystem.B.dylib" 390//go:cgo_import_dynamic libc_pthread_create pthread_create "/usr/lib/libSystem.B.dylib" 391//go:cgo_import_dynamic libc_exit exit "/usr/lib/libSystem.B.dylib" 392//go:cgo_import_dynamic libc_raise raise "/usr/lib/libSystem.B.dylib" 393 394//go:cgo_import_dynamic libc_open open "/usr/lib/libSystem.B.dylib" 395//go:cgo_import_dynamic libc_close close "/usr/lib/libSystem.B.dylib" 396//go:cgo_import_dynamic libc_read read "/usr/lib/libSystem.B.dylib" 397//go:cgo_import_dynamic libc_write write "/usr/lib/libSystem.B.dylib" 398 399//go:cgo_import_dynamic libc_mmap mmap "/usr/lib/libSystem.B.dylib" 400//go:cgo_import_dynamic libc_munmap munmap "/usr/lib/libSystem.B.dylib" 401//go:cgo_import_dynamic libc_madvise madvise "/usr/lib/libSystem.B.dylib" 402//go:cgo_import_dynamic libc_error __error "/usr/lib/libSystem.B.dylib" 403//go:cgo_import_dynamic libc_usleep usleep "/usr/lib/libSystem.B.dylib" 404 405//go:cgo_import_dynamic libc_mach_timebase_info mach_timebase_info "/usr/lib/libSystem.B.dylib" 406//go:cgo_import_dynamic libc_mach_absolute_time mach_absolute_time "/usr/lib/libSystem.B.dylib" 407//go:cgo_import_dynamic libc_gettimeofday gettimeofday "/usr/lib/libSystem.B.dylib" 408//go:cgo_import_dynamic libc_sigaction sigaction "/usr/lib/libSystem.B.dylib" 409//go:cgo_import_dynamic libc_pthread_sigmask pthread_sigmask "/usr/lib/libSystem.B.dylib" 410//go:cgo_import_dynamic libc_sigaltstack sigaltstack "/usr/lib/libSystem.B.dylib" 411//go:cgo_import_dynamic libc_getpid getpid "/usr/lib/libSystem.B.dylib" 412//go:cgo_import_dynamic libc_kill kill "/usr/lib/libSystem.B.dylib" 413//go:cgo_import_dynamic libc_setitimer setitimer "/usr/lib/libSystem.B.dylib" 414//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" 415//go:cgo_import_dynamic libc_fcntl fcntl "/usr/lib/libSystem.B.dylib" 416//go:cgo_import_dynamic libc_kqueue kqueue "/usr/lib/libSystem.B.dylib" 417//go:cgo_import_dynamic libc_kevent kevent "/usr/lib/libSystem.B.dylib" 418 419//go:cgo_import_dynamic libc_pthread_mutex_init pthread_mutex_init "/usr/lib/libSystem.B.dylib" 420//go:cgo_import_dynamic libc_pthread_mutex_lock pthread_mutex_lock "/usr/lib/libSystem.B.dylib" 421//go:cgo_import_dynamic libc_pthread_mutex_unlock pthread_mutex_unlock "/usr/lib/libSystem.B.dylib" 422//go:cgo_import_dynamic libc_pthread_cond_init pthread_cond_init "/usr/lib/libSystem.B.dylib" 423//go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" 424//go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" 425//go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" 426 427// Magic incantation to get libSystem actually dynamically linked. 428// TODO: Why does the code require this? See cmd/link/internal/ld/go.go 429//go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib" 430