1// Copyright 2014 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// +build ppc64 ppc64le 6 7#include "go_asm.h" 8#include "go_tls.h" 9#include "funcdata.h" 10#include "textflag.h" 11#include "asm_ppc64x.h" 12 13#ifdef GOOS_aix 14#define cgoCalleeStackSize 48 15#else 16#define cgoCalleeStackSize 32 17#endif 18 19TEXT runtime·rt0_go(SB),NOSPLIT,$0 20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer 21 22 // initialize essential registers 23 BL runtime·reginit(SB) 24 25 SUB $(FIXED_FRAME+16), R1 26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame 27 MOVW R3, FIXED_FRAME+0(R1) // argc 28 MOVD R4, FIXED_FRAME+8(R1) // argv 29 30 // create istack out of the given (operating system) stack. 31 // _cgo_init may update stackguard. 32 MOVD $runtime·g0(SB), g 33 BL runtime·save_g(SB) 34 MOVD $(-64*1024), R31 35 ADD R31, R1, R3 36 MOVD R3, g_stackguard0(g) 37 MOVD R3, g_stackguard1(g) 38 MOVD R3, (g_stack+stack_lo)(g) 39 MOVD R1, (g_stack+stack_hi)(g) 40 41 // if there is a _cgo_init, call it using the gcc ABI. 42 MOVD _cgo_init(SB), R12 43 CMP R0, R12 44 BEQ nocgo 45#ifdef GOARCH_ppc64 46 // ppc64 use elf ABI v1. we must get the real entry address from 47 // first slot of the function descriptor before call. 48 MOVD 8(R12), R2 49 MOVD (R12), R12 50#endif 51 MOVD R12, CTR // r12 = "global function entry point" 52 MOVD R13, R5 // arg 2: TLS base pointer 53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg 54 MOVD g, R3 // arg 0: G 55 // C functions expect 32 (48 for AIX) bytes of space on caller 56 // stack frame and a 16-byte aligned R1 57 MOVD R1, R14 // save current stack 58 SUB $cgoCalleeStackSize, R1 // reserve the callee area 59 RLDCR $0, R1, $~15, R1 // 16-byte align 60 BL (CTR) // may clobber R0, R3-R12 61 MOVD R14, R1 // restore stack 62#ifndef GOOS_aix 63 MOVD 24(R1), R2 64#endif 65 XOR R0, R0 // fix R0 66 67nocgo: 68 // update stackguard after _cgo_init 69 MOVD (g_stack+stack_lo)(g), R3 70 ADD $const__StackGuard, R3 71 MOVD R3, g_stackguard0(g) 72 MOVD R3, g_stackguard1(g) 73 74 // set the per-goroutine and per-mach "registers" 75 MOVD $runtime·m0(SB), R3 76 77 // save m->g0 = g0 78 MOVD g, m_g0(R3) 79 // save m0 to g0->m 80 MOVD R3, g_m(g) 81 82 BL runtime·check(SB) 83 84 // args are already prepared 85 BL runtime·args(SB) 86 BL runtime·osinit(SB) 87 BL runtime·schedinit(SB) 88 89 // create a new goroutine to start program 90 MOVD $runtime·mainPC(SB), R3 // entry 91 MOVDU R3, -8(R1) 92 MOVDU R0, -8(R1) 93 MOVDU R0, -8(R1) 94 MOVDU R0, -8(R1) 95 MOVDU R0, -8(R1) 96 MOVDU R0, -8(R1) 97 BL runtime·newproc(SB) 98 ADD $(16+FIXED_FRAME), R1 99 100 // start this M 101 BL runtime·mstart(SB) 102 103 MOVD R0, 0(R0) 104 RET 105 106DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 107GLOBL runtime·mainPC(SB),RODATA,$8 108 109TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 110 MOVD R0, 0(R0) // TODO: TD 111 RET 112 113TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 114 RET 115 116// Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64 117TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0 118 // crosscall_ppc64 and crosscall2 need to reginit, but can't 119 // get at the 'runtime.reginit' symbol. 120 BR runtime·reginit(SB) 121 122TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0 123 // set R0 to zero, it's expected by the toolchain 124 XOR R0, R0 125 RET 126 127/* 128 * go-routine 129 */ 130 131// void gosave(Gobuf*) 132// save state in Gobuf; setjmp 133TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8 134 MOVD buf+0(FP), R3 135 MOVD R1, gobuf_sp(R3) 136 MOVD LR, R31 137 MOVD R31, gobuf_pc(R3) 138 MOVD g, gobuf_g(R3) 139 MOVD R0, gobuf_lr(R3) 140 MOVD R0, gobuf_ret(R3) 141 // Assert ctxt is zero. See func save. 142 MOVD gobuf_ctxt(R3), R3 143 CMP R0, R3 144 BEQ 2(PC) 145 BL runtime·badctxt(SB) 146 RET 147 148// void gogo(Gobuf*) 149// restore state from Gobuf; longjmp 150TEXT runtime·gogo(SB), NOSPLIT, $16-8 151 MOVD buf+0(FP), R5 152 MOVD gobuf_g(R5), g // make sure g is not nil 153 BL runtime·save_g(SB) 154 155 MOVD 0(g), R4 156 MOVD gobuf_sp(R5), R1 157 MOVD gobuf_lr(R5), R31 158#ifndef GOOS_aix 159 MOVD 24(R1), R2 // restore R2 160#endif 161 MOVD R31, LR 162 MOVD gobuf_ret(R5), R3 163 MOVD gobuf_ctxt(R5), R11 164 MOVD R0, gobuf_sp(R5) 165 MOVD R0, gobuf_ret(R5) 166 MOVD R0, gobuf_lr(R5) 167 MOVD R0, gobuf_ctxt(R5) 168 CMP R0, R0 // set condition codes for == test, needed by stack split 169 MOVD gobuf_pc(R5), R12 170 MOVD R12, CTR 171 BR (CTR) 172 173// void mcall(fn func(*g)) 174// Switch to m->g0's stack, call fn(g). 175// Fn must never return. It should gogo(&g->sched) 176// to keep running g. 177TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 178 // Save caller state in g->sched 179 MOVD R1, (g_sched+gobuf_sp)(g) 180 MOVD LR, R31 181 MOVD R31, (g_sched+gobuf_pc)(g) 182 MOVD R0, (g_sched+gobuf_lr)(g) 183 MOVD g, (g_sched+gobuf_g)(g) 184 185 // Switch to m->g0 & its stack, call fn. 186 MOVD g, R3 187 MOVD g_m(g), R8 188 MOVD m_g0(R8), g 189 BL runtime·save_g(SB) 190 CMP g, R3 191 BNE 2(PC) 192 BR runtime·badmcall(SB) 193 MOVD fn+0(FP), R11 // context 194 MOVD 0(R11), R12 // code pointer 195 MOVD R12, CTR 196 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp 197 MOVDU R3, -8(R1) 198 MOVDU R0, -8(R1) 199 MOVDU R0, -8(R1) 200 MOVDU R0, -8(R1) 201 MOVDU R0, -8(R1) 202 BL (CTR) 203 MOVD 24(R1), R2 204 BR runtime·badmcall2(SB) 205 206// systemstack_switch is a dummy routine that systemstack leaves at the bottom 207// of the G stack. We need to distinguish the routine that 208// lives at the bottom of the G stack from the one that lives 209// at the top of the system stack because the one at the top of 210// the system stack terminates the stack walk (see topofstack()). 211TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 212 // We have several undefs here so that 16 bytes past 213 // $runtime·systemstack_switch lies within them whether or not the 214 // instructions that derive r2 from r12 are there. 215 UNDEF 216 UNDEF 217 UNDEF 218 BL (LR) // make sure this function is not leaf 219 RET 220 221// func systemstack(fn func()) 222TEXT runtime·systemstack(SB), NOSPLIT, $0-8 223 MOVD fn+0(FP), R3 // R3 = fn 224 MOVD R3, R11 // context 225 MOVD g_m(g), R4 // R4 = m 226 227 MOVD m_gsignal(R4), R5 // R5 = gsignal 228 CMP g, R5 229 BEQ noswitch 230 231 MOVD m_g0(R4), R5 // R5 = g0 232 CMP g, R5 233 BEQ noswitch 234 235 MOVD m_curg(R4), R6 236 CMP g, R6 237 BEQ switch 238 239 // Bad: g is not gsignal, not g0, not curg. What is it? 240 // Hide call from linker nosplit analysis. 241 MOVD $runtime·badsystemstack(SB), R12 242 MOVD R12, CTR 243 BL (CTR) 244 BL runtime·abort(SB) 245 246switch: 247 // save our state in g->sched. Pretend to 248 // be systemstack_switch if the G stack is scanned. 249 MOVD $runtime·systemstack_switch(SB), R6 250 ADD $16, R6 // get past prologue (including r2-setting instructions when they're there) 251 MOVD R6, (g_sched+gobuf_pc)(g) 252 MOVD R1, (g_sched+gobuf_sp)(g) 253 MOVD R0, (g_sched+gobuf_lr)(g) 254 MOVD g, (g_sched+gobuf_g)(g) 255 256 // switch to g0 257 MOVD R5, g 258 BL runtime·save_g(SB) 259 MOVD (g_sched+gobuf_sp)(g), R3 260 // make it look like mstart called systemstack on g0, to stop traceback 261 SUB $FIXED_FRAME, R3 262 MOVD $runtime·mstart(SB), R4 263 MOVD R4, 0(R3) 264 MOVD R3, R1 265 266 // call target function 267 MOVD 0(R11), R12 // code pointer 268 MOVD R12, CTR 269 BL (CTR) 270 271 // restore TOC pointer. It seems unlikely that we will use systemstack 272 // to call a function defined in another module, but the results of 273 // doing so would be so confusing that it's worth doing this. 274 MOVD g_m(g), R3 275 MOVD m_curg(R3), g 276 MOVD (g_sched+gobuf_sp)(g), R3 277#ifndef GOOS_aix 278 MOVD 24(R3), R2 279#endif 280 // switch back to g 281 MOVD g_m(g), R3 282 MOVD m_curg(R3), g 283 BL runtime·save_g(SB) 284 MOVD (g_sched+gobuf_sp)(g), R1 285 MOVD R0, (g_sched+gobuf_sp)(g) 286 RET 287 288noswitch: 289 // already on m stack, just call directly 290 // On other arches we do a tail call here, but it appears to be 291 // impossible to tail call a function pointer in shared mode on 292 // ppc64 because the caller is responsible for restoring the TOC. 293 MOVD 0(R11), R12 // code pointer 294 MOVD R12, CTR 295 BL (CTR) 296#ifndef GOOS_aix 297 MOVD 24(R1), R2 298#endif 299 RET 300 301/* 302 * support for morestack 303 */ 304 305// Called during function prolog when more stack is needed. 306// Caller has already loaded: 307// R3: framesize, R4: argsize, R5: LR 308// 309// The traceback routines see morestack on a g0 as being 310// the top of a stack (for example, morestack calling newstack 311// calling the scheduler calling newm calling gc), so we must 312// record an argument size. For that purpose, it has no arguments. 313TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 314 // Cannot grow scheduler stack (m->g0). 315 MOVD g_m(g), R7 316 MOVD m_g0(R7), R8 317 CMP g, R8 318 BNE 3(PC) 319 BL runtime·badmorestackg0(SB) 320 BL runtime·abort(SB) 321 322 // Cannot grow signal stack (m->gsignal). 323 MOVD m_gsignal(R7), R8 324 CMP g, R8 325 BNE 3(PC) 326 BL runtime·badmorestackgsignal(SB) 327 BL runtime·abort(SB) 328 329 // Called from f. 330 // Set g->sched to context in f. 331 MOVD R1, (g_sched+gobuf_sp)(g) 332 MOVD LR, R8 333 MOVD R8, (g_sched+gobuf_pc)(g) 334 MOVD R5, (g_sched+gobuf_lr)(g) 335 MOVD R11, (g_sched+gobuf_ctxt)(g) 336 337 // Called from f. 338 // Set m->morebuf to f's caller. 339 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 340 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 341 MOVD g, (m_morebuf+gobuf_g)(R7) 342 343 // Call newstack on m->g0's stack. 344 MOVD m_g0(R7), g 345 BL runtime·save_g(SB) 346 MOVD (g_sched+gobuf_sp)(g), R1 347 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0 348 BL runtime·newstack(SB) 349 350 // Not reached, but make sure the return PC from the call to newstack 351 // is still in this function, and not the beginning of the next. 352 UNDEF 353 354TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 355 MOVD R0, R11 356 BR runtime·morestack(SB) 357 358// reflectcall: call a function with the given argument list 359// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 360// we don't have variable-sized frames, so we use a small number 361// of constant-sized-frame functions to encode a few bits of size in the pc. 362// Caution: ugly multiline assembly macros in your future! 363 364#define DISPATCH(NAME,MAXSIZE) \ 365 MOVD $MAXSIZE, R31; \ 366 CMP R3, R31; \ 367 BGT 4(PC); \ 368 MOVD $NAME(SB), R12; \ 369 MOVD R12, CTR; \ 370 BR (CTR) 371// Note: can't just "BR NAME(SB)" - bad inlining results. 372 373TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32 374 MOVWZ argsize+24(FP), R3 375 DISPATCH(runtime·call32, 32) 376 DISPATCH(runtime·call64, 64) 377 DISPATCH(runtime·call128, 128) 378 DISPATCH(runtime·call256, 256) 379 DISPATCH(runtime·call512, 512) 380 DISPATCH(runtime·call1024, 1024) 381 DISPATCH(runtime·call2048, 2048) 382 DISPATCH(runtime·call4096, 4096) 383 DISPATCH(runtime·call8192, 8192) 384 DISPATCH(runtime·call16384, 16384) 385 DISPATCH(runtime·call32768, 32768) 386 DISPATCH(runtime·call65536, 65536) 387 DISPATCH(runtime·call131072, 131072) 388 DISPATCH(runtime·call262144, 262144) 389 DISPATCH(runtime·call524288, 524288) 390 DISPATCH(runtime·call1048576, 1048576) 391 DISPATCH(runtime·call2097152, 2097152) 392 DISPATCH(runtime·call4194304, 4194304) 393 DISPATCH(runtime·call8388608, 8388608) 394 DISPATCH(runtime·call16777216, 16777216) 395 DISPATCH(runtime·call33554432, 33554432) 396 DISPATCH(runtime·call67108864, 67108864) 397 DISPATCH(runtime·call134217728, 134217728) 398 DISPATCH(runtime·call268435456, 268435456) 399 DISPATCH(runtime·call536870912, 536870912) 400 DISPATCH(runtime·call1073741824, 1073741824) 401 MOVD $runtime·badreflectcall(SB), R12 402 MOVD R12, CTR 403 BR (CTR) 404 405#define CALLFN(NAME,MAXSIZE) \ 406TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 407 NO_LOCAL_POINTERS; \ 408 /* copy arguments to stack */ \ 409 MOVD arg+16(FP), R3; \ 410 MOVWZ argsize+24(FP), R4; \ 411 MOVD R1, R5; \ 412 CMP R4, $8; \ 413 BLT tailsetup; \ 414 /* copy 8 at a time if possible */ \ 415 ADD $(FIXED_FRAME-8), R5; \ 416 SUB $8, R3; \ 417top: \ 418 MOVDU 8(R3), R7; \ 419 MOVDU R7, 8(R5); \ 420 SUB $8, R4; \ 421 CMP R4, $8; \ 422 BGE top; \ 423 /* handle remaining bytes */ \ 424 CMP $0, R4; \ 425 BEQ callfn; \ 426 ADD $7, R3; \ 427 ADD $7, R5; \ 428 BR tail; \ 429tailsetup: \ 430 CMP $0, R4; \ 431 BEQ callfn; \ 432 ADD $(FIXED_FRAME-1), R5; \ 433 SUB $1, R3; \ 434tail: \ 435 MOVBU 1(R3), R6; \ 436 MOVBU R6, 1(R5); \ 437 SUB $1, R4; \ 438 CMP $0, R4; \ 439 BGT tail; \ 440callfn: \ 441 /* call function */ \ 442 MOVD f+8(FP), R11; \ 443#ifdef GOOS_aix \ 444 /* AIX won't trigger a SIGSEGV if R11 = nil */ \ 445 /* So it manually triggers it */ \ 446 CMP R0, R11 \ 447 BNE 2(PC) \ 448 MOVD R0, 0(R0) \ 449#endif \ 450 MOVD (R11), R12; \ 451 MOVD R12, CTR; \ 452 PCDATA $PCDATA_StackMapIndex, $0; \ 453 BL (CTR); \ 454#ifndef GOOS_aix \ 455 MOVD 24(R1), R2; \ 456#endif \ 457 /* copy return values back */ \ 458 MOVD argtype+0(FP), R7; \ 459 MOVD arg+16(FP), R3; \ 460 MOVWZ n+24(FP), R4; \ 461 MOVWZ retoffset+28(FP), R6; \ 462 ADD $FIXED_FRAME, R1, R5; \ 463 ADD R6, R5; \ 464 ADD R6, R3; \ 465 SUB R6, R4; \ 466 BL callRet<>(SB); \ 467 RET 468 469// callRet copies return values back at the end of call*. This is a 470// separate function so it can allocate stack space for the arguments 471// to reflectcallmove. It does not follow the Go ABI; it expects its 472// arguments in registers. 473TEXT callRet<>(SB), NOSPLIT, $32-0 474 MOVD R7, FIXED_FRAME+0(R1) 475 MOVD R3, FIXED_FRAME+8(R1) 476 MOVD R5, FIXED_FRAME+16(R1) 477 MOVD R4, FIXED_FRAME+24(R1) 478 BL runtime·reflectcallmove(SB) 479 RET 480 481CALLFN(·call32, 32) 482CALLFN(·call64, 64) 483CALLFN(·call128, 128) 484CALLFN(·call256, 256) 485CALLFN(·call512, 512) 486CALLFN(·call1024, 1024) 487CALLFN(·call2048, 2048) 488CALLFN(·call4096, 4096) 489CALLFN(·call8192, 8192) 490CALLFN(·call16384, 16384) 491CALLFN(·call32768, 32768) 492CALLFN(·call65536, 65536) 493CALLFN(·call131072, 131072) 494CALLFN(·call262144, 262144) 495CALLFN(·call524288, 524288) 496CALLFN(·call1048576, 1048576) 497CALLFN(·call2097152, 2097152) 498CALLFN(·call4194304, 4194304) 499CALLFN(·call8388608, 8388608) 500CALLFN(·call16777216, 16777216) 501CALLFN(·call33554432, 33554432) 502CALLFN(·call67108864, 67108864) 503CALLFN(·call134217728, 134217728) 504CALLFN(·call268435456, 268435456) 505CALLFN(·call536870912, 536870912) 506CALLFN(·call1073741824, 1073741824) 507 508TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4 509 MOVW cycles+0(FP), R7 510 // POWER does not have a pause/yield instruction equivalent. 511 // Instead, we can lower the program priority by setting the 512 // Program Priority Register prior to the wait loop and set it 513 // back to default afterwards. On Linux, the default priority is 514 // medium-low. For details, see page 837 of the ISA 3.0. 515 OR R1, R1, R1 // Set PPR priority to low 516again: 517 SUB $1, R7 518 CMP $0, R7 519 BNE again 520 OR R6, R6, R6 // Set PPR priority back to medium-low 521 RET 522 523// void jmpdefer(fv, sp); 524// called from deferreturn. 525// 1. grab stored LR for caller 526// 2. sub 8 bytes to get back to either nop or toc reload before deferreturn 527// 3. BR to fn 528// When dynamically linking Go, it is not sufficient to rewind to the BL 529// deferreturn -- we might be jumping between modules and so we need to reset 530// the TOC pointer in r2. To do this, codegen inserts MOVD 24(R1), R2 *before* 531// the BL deferreturn and jmpdefer rewinds to that. 532TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 533 MOVD 0(R1), R31 534 SUB $8, R31 535 MOVD R31, LR 536 537 MOVD fv+0(FP), R11 538 MOVD argp+8(FP), R1 539 SUB $FIXED_FRAME, R1 540#ifdef GOOS_aix 541 // AIX won't trigger a SIGSEGV if R11 = nil 542 // So it manually triggers it 543 CMP R0, R11 544 BNE 2(PC) 545 MOVD R0, 0(R0) 546#endif 547 MOVD 0(R11), R12 548 MOVD R12, CTR 549 BR (CTR) 550 551// Save state of caller into g->sched. Smashes R31. 552TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 553 MOVD LR, R31 554 MOVD R31, (g_sched+gobuf_pc)(g) 555 MOVD R1, (g_sched+gobuf_sp)(g) 556 MOVD R0, (g_sched+gobuf_lr)(g) 557 MOVD R0, (g_sched+gobuf_ret)(g) 558 // Assert ctxt is zero. See func save. 559 MOVD (g_sched+gobuf_ctxt)(g), R31 560 CMP R0, R31 561 BEQ 2(PC) 562 BL runtime·badctxt(SB) 563 RET 564 565#ifdef GOOS_aix 566#define asmcgocallSaveOffset cgoCalleeStackSize + 8 567#else 568#define asmcgocallSaveOffset cgoCalleeStackSize 569#endif 570 571// func asmcgocall(fn, arg unsafe.Pointer) int32 572// Call fn(arg) on the scheduler stack, 573// aligned appropriately for the gcc ABI. 574// See cgocall.go for more details. 575TEXT ·asmcgocall(SB),NOSPLIT,$0-20 576 MOVD fn+0(FP), R3 577 MOVD arg+8(FP), R4 578 579 MOVD R1, R7 // save original stack pointer 580 MOVD g, R5 581 582 // Figure out if we need to switch to m->g0 stack. 583 // We get called to create new OS threads too, and those 584 // come in on the m->g0 stack already. 585 // Moreover, if it's called inside the signal handler, it must not switch 586 // to g0 as it can be in use by another syscall. 587 MOVD g_m(g), R8 588 MOVD m_gsignal(R8), R6 589 CMP R6, g 590 BEQ g0 591 MOVD m_g0(R8), R6 592 CMP R6, g 593 BEQ g0 594 BL gosave<>(SB) 595 MOVD R6, g 596 BL runtime·save_g(SB) 597 MOVD (g_sched+gobuf_sp)(g), R1 598 599 // Now on a scheduling stack (a pthread-created stack). 600g0: 601#ifdef GOOS_aix 602 // Create a fake LR to improve backtrace. 603 MOVD $runtime·asmcgocall(SB), R6 604 MOVD R6, 16(R1) 605 // AIX also save one argument on the stack. 606 SUB $8, R1 607#endif 608 // Save room for two of our pointers, plus the callee 609 // save area that lives on the caller stack. 610 SUB $(asmcgocallSaveOffset+16), R1 611 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI 612 MOVD R5, (asmcgocallSaveOffset+8)(R1)// save old g on stack 613 MOVD (g_stack+stack_hi)(R5), R5 614 SUB R7, R5 615 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 616#ifdef GOOS_aix 617 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb 618#else 619 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?) 620#endif 621 // This is a "global call", so put the global entry point in r12 622 MOVD R3, R12 623 624#ifdef GOARCH_ppc64 625 // ppc64 use elf ABI v1. we must get the real entry address from 626 // first slot of the function descriptor before call. 627 // Same for AIX. 628 MOVD 8(R12), R2 629 MOVD (R12), R12 630#endif 631 MOVD R12, CTR 632 MOVD R4, R3 // arg in r3 633 BL (CTR) 634 // C code can clobber R0, so set it back to 0. F27-F31 are 635 // callee save, so we don't need to recover those. 636 XOR R0, R0 637 // Restore g, stack pointer, toc pointer. 638 // R3 is errno, so don't touch it 639 MOVD (asmcgocallSaveOffset+8)(R1), g 640 MOVD (g_stack+stack_hi)(g), R5 641 MOVD asmcgocallSaveOffset(R1), R6 642 SUB R6, R5 643#ifndef GOOS_aix 644 MOVD 24(R5), R2 645#endif 646 MOVD R5, R1 647 BL runtime·save_g(SB) 648 649 MOVW R3, ret+16(FP) 650 RET 651 652// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 653// Turn the fn into a Go func (by taking its address) and call 654// cgocallback_gofunc. 655TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 656 MOVD $fn+0(FP), R3 657 MOVD R3, FIXED_FRAME+0(R1) 658 MOVD frame+8(FP), R3 659 MOVD R3, FIXED_FRAME+8(R1) 660 MOVD framesize+16(FP), R3 661 MOVD R3, FIXED_FRAME+16(R1) 662 MOVD ctxt+24(FP), R3 663 MOVD R3, FIXED_FRAME+24(R1) 664 MOVD $runtime·cgocallback_gofunc(SB), R12 665 MOVD R12, CTR 666 BL (CTR) 667 RET 668 669// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 670// See cgocall.go for more details. 671TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 672 NO_LOCAL_POINTERS 673 674 // Load m and g from thread-local storage. 675 MOVBZ runtime·iscgo(SB), R3 676 CMP R3, $0 677 BEQ nocgo 678 BL runtime·load_g(SB) 679nocgo: 680 681 // If g is nil, Go did not create the current thread. 682 // Call needm to obtain one for temporary use. 683 // In this case, we're running on the thread stack, so there's 684 // lots of space, but the linker doesn't know. Hide the call from 685 // the linker analysis by using an indirect call. 686 CMP g, $0 687 BEQ needm 688 689 MOVD g_m(g), R8 690 MOVD R8, savedm-8(SP) 691 BR havem 692 693needm: 694 MOVD g, savedm-8(SP) // g is zero, so is m. 695 MOVD $runtime·needm(SB), R12 696 MOVD R12, CTR 697 BL (CTR) 698 699 // Set m->sched.sp = SP, so that if a panic happens 700 // during the function we are about to execute, it will 701 // have a valid SP to run on the g0 stack. 702 // The next few lines (after the havem label) 703 // will save this SP onto the stack and then write 704 // the same SP back to m->sched.sp. That seems redundant, 705 // but if an unrecovered panic happens, unwindm will 706 // restore the g->sched.sp from the stack location 707 // and then systemstack will try to use it. If we don't set it here, 708 // that restored SP will be uninitialized (typically 0) and 709 // will not be usable. 710 MOVD g_m(g), R8 711 MOVD m_g0(R8), R3 712 MOVD R1, (g_sched+gobuf_sp)(R3) 713 714havem: 715 // Now there's a valid m, and we're running on its m->g0. 716 // Save current m->g0->sched.sp on stack and then set it to SP. 717 // Save current sp in m->g0->sched.sp in preparation for 718 // switch back to m->curg stack. 719 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 720 MOVD m_g0(R8), R3 721 MOVD (g_sched+gobuf_sp)(R3), R4 722 MOVD R4, savedsp-16(SP) 723 MOVD R1, (g_sched+gobuf_sp)(R3) 724 725 // Switch to m->curg stack and call runtime.cgocallbackg. 726 // Because we are taking over the execution of m->curg 727 // but *not* resuming what had been running, we need to 728 // save that information (m->curg->sched) so we can restore it. 729 // We can restore m->curg->sched.sp easily, because calling 730 // runtime.cgocallbackg leaves SP unchanged upon return. 731 // To save m->curg->sched.pc, we push it onto the stack. 732 // This has the added benefit that it looks to the traceback 733 // routine like cgocallbackg is going to return to that 734 // PC (because the frame we allocate below has the same 735 // size as cgocallback_gofunc's frame declared above) 736 // so that the traceback will seamlessly trace back into 737 // the earlier calls. 738 // 739 // In the new goroutine, -8(SP) is unused (where SP refers to 740 // m->curg's SP while we're setting it up, before we've adjusted it). 741 MOVD m_curg(R8), g 742 BL runtime·save_g(SB) 743 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 744 MOVD (g_sched+gobuf_pc)(g), R5 745 MOVD R5, -(FIXED_FRAME+16)(R4) 746 MOVD ctxt+24(FP), R3 747 MOVD R3, -16(R4) 748 MOVD $-(FIXED_FRAME+16)(R4), R1 749 BL runtime·cgocallbackg(SB) 750 751 // Restore g->sched (== m->curg->sched) from saved values. 752 MOVD 0(R1), R5 753 MOVD R5, (g_sched+gobuf_pc)(g) 754 MOVD $(FIXED_FRAME+16)(R1), R4 755 MOVD R4, (g_sched+gobuf_sp)(g) 756 757 // Switch back to m->g0's stack and restore m->g0->sched.sp. 758 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 759 // so we do not have to restore it.) 760 MOVD g_m(g), R8 761 MOVD m_g0(R8), g 762 BL runtime·save_g(SB) 763 MOVD (g_sched+gobuf_sp)(g), R1 764 MOVD savedsp-16(SP), R4 765 MOVD R4, (g_sched+gobuf_sp)(g) 766 767 // If the m on entry was nil, we called needm above to borrow an m 768 // for the duration of the call. Since the call is over, return it with dropm. 769 MOVD savedm-8(SP), R6 770 CMP R6, $0 771 BNE droppedm 772 MOVD $runtime·dropm(SB), R12 773 MOVD R12, CTR 774 BL (CTR) 775droppedm: 776 777 // Done! 778 RET 779 780// void setg(G*); set g. for use by needm. 781TEXT runtime·setg(SB), NOSPLIT, $0-8 782 MOVD gg+0(FP), g 783 // This only happens if iscgo, so jump straight to save_g 784 BL runtime·save_g(SB) 785 RET 786 787#ifdef GOARCH_ppc64 788#ifdef GOOS_aix 789DATA setg_gcc<>+0(SB)/8, $_setg_gcc<>(SB) 790DATA setg_gcc<>+8(SB)/8, $TOC(SB) 791DATA setg_gcc<>+16(SB)/8, $0 792GLOBL setg_gcc<>(SB), NOPTR, $24 793#else 794TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 795 DWORD $_setg_gcc<>(SB) 796 DWORD $0 797 DWORD $0 798#endif 799#endif 800 801// void setg_gcc(G*); set g in C TLS. 802// Must obey the gcc calling convention. 803#ifdef GOARCH_ppc64le 804TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 805#else 806TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 807#endif 808 // The standard prologue clobbers R31, which is callee-save in 809 // the C ABI, so we have to use $-8-0 and save LR ourselves. 810 MOVD LR, R4 811 // Also save g and R31, since they're callee-save in C ABI 812 MOVD R31, R5 813 MOVD g, R6 814 815 MOVD R3, g 816 BL runtime·save_g(SB) 817 818 MOVD R6, g 819 MOVD R5, R31 820 MOVD R4, LR 821 RET 822 823TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 824 MOVW (R0), R0 825 UNDEF 826 827#define TBR 268 828 829// int64 runtime·cputicks(void) 830TEXT runtime·cputicks(SB),NOSPLIT,$0-8 831 MOVD SPR(TBR), R3 832 MOVD R3, ret+0(FP) 833 RET 834 835// AES hashing not implemented for ppc64 836TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32 837 JMP runtime·memhashFallback(SB) 838TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24 839 JMP runtime·strhashFallback(SB) 840TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 841 JMP runtime·memhash32Fallback(SB) 842TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 843 JMP runtime·memhash64Fallback(SB) 844 845TEXT runtime·return0(SB), NOSPLIT, $0 846 MOVW $0, R3 847 RET 848 849// Called from cgo wrappers, this function returns g->m->curg.stack.hi. 850// Must obey the gcc calling convention. 851#ifdef GOOS_aix 852// On AIX, _cgo_topofstack is defined in runtime/cgo, because it must 853// be a longcall in order to prevent trampolines from ld. 854TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 855#else 856TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 857#endif 858 // g (R30) and R31 are callee-save in the C ABI, so save them 859 MOVD g, R4 860 MOVD R31, R5 861 MOVD LR, R6 862 863 BL runtime·load_g(SB) // clobbers g (R30), R31 864 MOVD g_m(g), R3 865 MOVD m_curg(R3), R3 866 MOVD (g_stack+stack_hi)(R3), R3 867 868 MOVD R4, g 869 MOVD R5, R31 870 MOVD R6, LR 871 RET 872 873// The top-most function running on a goroutine 874// returns to goexit+PCQuantum. 875// 876// When dynamically linking Go, it can be returned to from a function 877// implemented in a different module and so needs to reload the TOC pointer 878// from the stack (although this function declares that it does not set up x-a 879// frame, newproc1 does in fact allocate one for goexit and saves the TOC 880// pointer in the correct place). 881// goexit+_PCQuantum is halfway through the usual global entry point prologue 882// that derives r2 from r12 which is a bit silly, but not harmful. 883TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 884 MOVD 24(R1), R2 885 BL runtime·goexit1(SB) // does not return 886 // traceback from goexit1 must hit code range of goexit 887 MOVD R0, R0 // NOP 888 889// prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the 890// module containing runtime) to the frame that goexit will execute in when 891// the goroutine exits. It's implemented in assembly mainly because that's the 892// easiest way to get access to R2. 893TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8 894 MOVD sp+0(FP), R3 895 MOVD R2, 24(R3) 896 RET 897 898TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0 899 ADD $-8, R1 900 MOVD R31, 0(R1) 901 MOVD runtime·lastmoduledatap(SB), R4 902 MOVD R3, moduledata_next(R4) 903 MOVD R3, runtime·lastmoduledatap(SB) 904 MOVD 0(R1), R31 905 ADD $8, R1 906 RET 907 908TEXT ·checkASM(SB),NOSPLIT,$0-1 909 MOVW $1, R3 910 MOVB R3, ret+0(FP) 911 RET 912 913// gcWriteBarrier performs a heap pointer write and informs the GC. 914// 915// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: 916// - R20 is the destination of the write 917// - R21 is the value being written at R20. 918// It clobbers condition codes. 919// It does not clobber R0 through R15, 920// but may clobber any other register, *including* R31. 921TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 922 // The standard prologue clobbers R31. 923 // We use R16 and R17 as scratch registers. 924 MOVD g_m(g), R16 925 MOVD m_p(R16), R16 926 MOVD (p_wbBuf+wbBuf_next)(R16), R17 927 // Increment wbBuf.next position. 928 ADD $16, R17 929 MOVD R17, (p_wbBuf+wbBuf_next)(R16) 930 MOVD (p_wbBuf+wbBuf_end)(R16), R16 931 CMP R16, R17 932 // Record the write. 933 MOVD R21, -16(R17) // Record value 934 MOVD (R20), R16 // TODO: This turns bad writes into bad reads. 935 MOVD R16, -8(R17) // Record *slot 936 // Is the buffer full? (flags set in CMP above) 937 BEQ flush 938ret: 939 // Do the write. 940 MOVD R21, (R20) 941 RET 942 943flush: 944 // Save registers R0 through R15 since these were not saved by the caller. 945 // We don't save all registers on ppc64 because it takes too much space. 946 MOVD R20, (FIXED_FRAME+0)(R1) // Also first argument to wbBufFlush 947 MOVD R21, (FIXED_FRAME+8)(R1) // Also second argument to wbBufFlush 948 // R0 is always 0, so no need to spill. 949 // R1 is SP. 950 // R2 is SB. 951 MOVD R3, (FIXED_FRAME+16)(R1) 952 MOVD R4, (FIXED_FRAME+24)(R1) 953 MOVD R5, (FIXED_FRAME+32)(R1) 954 MOVD R6, (FIXED_FRAME+40)(R1) 955 MOVD R7, (FIXED_FRAME+48)(R1) 956 MOVD R8, (FIXED_FRAME+56)(R1) 957 MOVD R9, (FIXED_FRAME+64)(R1) 958 MOVD R10, (FIXED_FRAME+72)(R1) 959 MOVD R11, (FIXED_FRAME+80)(R1) 960 MOVD R12, (FIXED_FRAME+88)(R1) 961 // R13 is REGTLS 962 MOVD R14, (FIXED_FRAME+96)(R1) 963 MOVD R15, (FIXED_FRAME+104)(R1) 964 965 // This takes arguments R20 and R21. 966 CALL runtime·wbBufFlush(SB) 967 968 MOVD (FIXED_FRAME+0)(R1), R20 969 MOVD (FIXED_FRAME+8)(R1), R21 970 MOVD (FIXED_FRAME+16)(R1), R3 971 MOVD (FIXED_FRAME+24)(R1), R4 972 MOVD (FIXED_FRAME+32)(R1), R5 973 MOVD (FIXED_FRAME+40)(R1), R6 974 MOVD (FIXED_FRAME+48)(R1), R7 975 MOVD (FIXED_FRAME+56)(R1), R8 976 MOVD (FIXED_FRAME+64)(R1), R9 977 MOVD (FIXED_FRAME+72)(R1), R10 978 MOVD (FIXED_FRAME+80)(R1), R11 979 MOVD (FIXED_FRAME+88)(R1), R12 980 MOVD (FIXED_FRAME+96)(R1), R14 981 MOVD (FIXED_FRAME+104)(R1), R15 982 JMP ret 983 984// Note: these functions use a special calling convention to save generated code space. 985// Arguments are passed in registers, but the space for those arguments are allocated 986// in the caller's stack frame. These stubs write the args into that stack space and 987// then tail call to the corresponding runtime handler. 988// The tail call makes these stubs disappear in backtraces. 989TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 990 MOVD R3, x+0(FP) 991 MOVD R4, y+8(FP) 992 JMP runtime·goPanicIndex(SB) 993TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 994 MOVD R3, x+0(FP) 995 MOVD R4, y+8(FP) 996 JMP runtime·goPanicIndexU(SB) 997TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 998 MOVD R4, x+0(FP) 999 MOVD R5, y+8(FP) 1000 JMP runtime·goPanicSliceAlen(SB) 1001TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 1002 MOVD R4, x+0(FP) 1003 MOVD R5, y+8(FP) 1004 JMP runtime·goPanicSliceAlenU(SB) 1005TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 1006 MOVD R4, x+0(FP) 1007 MOVD R5, y+8(FP) 1008 JMP runtime·goPanicSliceAcap(SB) 1009TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 1010 MOVD R4, x+0(FP) 1011 MOVD R5, y+8(FP) 1012 JMP runtime·goPanicSliceAcapU(SB) 1013TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 1014 MOVD R3, x+0(FP) 1015 MOVD R4, y+8(FP) 1016 JMP runtime·goPanicSliceB(SB) 1017TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 1018 MOVD R3, x+0(FP) 1019 MOVD R4, y+8(FP) 1020 JMP runtime·goPanicSliceBU(SB) 1021TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 1022 MOVD R5, x+0(FP) 1023 MOVD R6, y+8(FP) 1024 JMP runtime·goPanicSlice3Alen(SB) 1025TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 1026 MOVD R5, x+0(FP) 1027 MOVD R6, y+8(FP) 1028 JMP runtime·goPanicSlice3AlenU(SB) 1029TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 1030 MOVD R5, x+0(FP) 1031 MOVD R6, y+8(FP) 1032 JMP runtime·goPanicSlice3Acap(SB) 1033TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 1034 MOVD R5, x+0(FP) 1035 MOVD R6, y+8(FP) 1036 JMP runtime·goPanicSlice3AcapU(SB) 1037TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 1038 MOVD R4, x+0(FP) 1039 MOVD R5, y+8(FP) 1040 JMP runtime·goPanicSlice3B(SB) 1041TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 1042 MOVD R4, x+0(FP) 1043 MOVD R5, y+8(FP) 1044 JMP runtime·goPanicSlice3BU(SB) 1045TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 1046 MOVD R3, x+0(FP) 1047 MOVD R4, y+8(FP) 1048 JMP runtime·goPanicSlice3C(SB) 1049TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 1050 MOVD R3, x+0(FP) 1051 MOVD R4, y+8(FP) 1052 JMP runtime·goPanicSlice3CU(SB) 1053