1/* $NetBSD: locore.s,v 1.332 2010/12/20 00:25:44 matt Exp $ */ 2 3/* 4 * Copyright (c) 2006-2010 Matthew R. Green 5 * Copyright (c) 1996-2002 Eduardo Horvath 6 * Copyright (c) 1996 Paul Kranenburg 7 * Copyright (c) 1996 8 * The President and Fellows of Harvard College. 9 * All rights reserved. 10 * Copyright (c) 1992, 1993 11 * The Regents of the University of California. 12 * All rights reserved. 13 * 14 * This software was developed by the Computer Systems Engineering group 15 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 16 * contributed to Berkeley. 17 * 18 * All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the University of 21 * California, Lawrence Berkeley Laboratory. 22 * This product includes software developed by Harvard University. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 1. Redistributions of source code must retain the above copyright 28 * notice, this list of conditions and the following disclaimer. 29 * 2. Redistributions in binary form must reproduce the above copyright 30 * notice, this list of conditions and the following disclaimer in the 31 * documentation and/or other materials provided with the 32 * distribution. 33 * 3. All advertising materials mentioning features or use of this 34 * software must display the following acknowledgement: 35 * This product includes software developed by the University of 36 * California, Berkeley and its contributors. 37 * This product includes software developed by Harvard University. 38 * This product includes software developed by Paul Kranenburg. 39 * 4. Neither the name of the University nor the names of its 40 * contributors may be used to endorse or promote products derived 41 * from this software without specific prior written permission. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' 44 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 45 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 46 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 47 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 48 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 49 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 50 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 51 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 52 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 53 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 54 * DAMAGE. 55 * 56 * @(#)locore.s 8.4 (Berkeley) 12/10/93 57 */ 58 59#undef PARANOID /* Extremely expensive consistency checks */ 60#undef NO_VCACHE /* Map w/D$ disabled */ 61#undef TRAPSTATS /* Count traps */ 62#undef TRAPS_USE_IG /* Use Interrupt Globals for all traps */ 63#define HWREF /* Track ref/mod bits in trap handlers */ 64#undef DCACHE_BUG /* Flush D$ around ASI_PHYS accesses */ 65#undef NO_TSB /* Don't use TSB */ 66#define BB_ERRATA_1 /* writes to TICK_CMPR may fail */ 67#undef TLB_FLUSH_LOWVA /* also flush 32-bit entries from the MMU */ 68 69#include "opt_ddb.h" 70#include "opt_kgdb.h" 71#include "opt_multiprocessor.h" 72#include "opt_compat_netbsd.h" 73#include "opt_compat_netbsd32.h" 74#include "opt_lockdebug.h" 75 76#include "assym.h" 77#include <machine/param.h> 78#include <sparc64/sparc64/intreg.h> 79#include <sparc64/sparc64/timerreg.h> 80#include <machine/ctlreg.h> 81#include <machine/psl.h> 82#include <machine/signal.h> 83#include <machine/trap.h> 84#include <machine/frame.h> 85#include <machine/pte.h> 86#include <machine/pmap.h> 87#include <machine/intr.h> 88#include <machine/asm.h> 89#include <machine/locore.h> 90#include <sys/syscall.h> 91 92#include "ksyms.h" 93 94#if 1 95/* 96 * Try to issue an elf note to ask the Solaris 97 * bootloader to align the kernel properly. 98 */ 99 .section .note 100 .word 0x0d 101 .word 4 ! Dunno why 102 .word 1 1030: .asciz "SUNW Solaris" 1041: 105 .align 4 106 .word 0x0400000 107#endif 108 109 .register %g2,#scratch 110 .register %g3,#scratch 111 112 113 .data 114 .globl _C_LABEL(data_start) 115_C_LABEL(data_start): ! Start of data segment 116 117#ifdef KGDB 118/* 119 * Another item that must be aligned, easiest to put it here. 120 */ 121KGDB_STACK_SIZE = 2048 122 .globl _C_LABEL(kgdb_stack) 123_C_LABEL(kgdb_stack): 124 .space KGDB_STACK_SIZE ! hope this is enough 125#endif 126 127#ifdef NOTDEF_DEBUG 128/* 129 * This stack is used when we detect kernel stack corruption. 130 */ 131 .space USPACE 132 .align 16 133panicstack: 134#endif 135 136/* 137 * romp is the prom entry pointer 138 * romtba is the prom trap table base address 139 */ 140 .globl romp 141romp: POINTER 0 142 .globl romtba 143romtba: POINTER 0 144 145 _ALIGN 146 .text 147 148/* 149 * The v9 trap frame is stored in the special trap registers. The 150 * register window is only modified on window overflow, underflow, 151 * and clean window traps, where it points to the register window 152 * needing service. Traps have space for 8 instructions, except for 153 * the window overflow, underflow, and clean window traps which are 154 * 32 instructions long, large enough to in-line. 155 * 156 * The spitfire CPU (Ultra I) has 4 different sets of global registers. 157 * (blah blah...) 158 * 159 * I used to generate these numbers by address arithmetic, but gas's 160 * expression evaluator has about as much sense as your average slug 161 * (oddly enough, the code looks about as slimy too). Thus, all the 162 * trap numbers are given as arguments to the trap macros. This means 163 * there is one line per trap. Sigh. 164 * 165 * Hardware interrupt vectors can be `linked'---the linkage is to regular 166 * C code---or rewired to fast in-window handlers. The latter are good 167 * for unbuffered hardware like the Zilog serial chip and the AMD audio 168 * chip, where many interrupts can be handled trivially with pseudo-DMA 169 * or similar. Only one `fast' interrupt can be used per level, however, 170 * and direct and `fast' interrupts are incompatible. Routines in intr.c 171 * handle setting these, with optional paranoia. 172 */ 173 174/* 175 * TA8 -- trap align for 8 instruction traps 176 * TA32 -- trap align for 32 instruction traps 177 */ 178#define TA8 .align 32 179#define TA32 .align 128 180 181/* 182 * v9 trap macros: 183 * 184 * We have a problem with v9 traps; we have no registers to put the 185 * trap type into. But we do have a %tt register which already has 186 * that information. Trap types in these macros are all dummys. 187 */ 188 /* regular vectored traps */ 189 190#define VTRAP(type, label) \ 191 ba,a,pt %icc,label; nop; NOTREACHED; TA8 192 193 /* hardware interrupts (can be linked or made `fast') */ 194#define HARDINT4U(lev) \ 195 VTRAP(lev, _C_LABEL(sparc_interrupt)) 196 197 /* software interrupts (may not be made direct, sorry---but you 198 should not be using them trivially anyway) */ 199#define SOFTINT4U(lev, bit) \ 200 HARDINT4U(lev) 201 202 /* traps that just call trap() */ 203#define TRAP(type) VTRAP(type, slowtrap) 204 205 /* architecturally undefined traps (cause panic) */ 206#ifndef DEBUG 207#define UTRAP(type) sir; VTRAP(type, slowtrap) 208#else 209#define UTRAP(type) VTRAP(type, slowtrap) 210#endif 211 212 /* software undefined traps (may be replaced) */ 213#define STRAP(type) VTRAP(type, slowtrap) 214 215/* breakpoint acts differently under kgdb */ 216#ifdef KGDB 217#define BPT VTRAP(T_BREAKPOINT, bpt) 218#define BPT_KGDB_EXEC VTRAP(T_KGDB_EXEC, bpt) 219#else 220#define BPT TRAP(T_BREAKPOINT) 221#define BPT_KGDB_EXEC TRAP(T_KGDB_EXEC) 222#endif 223 224#define SYSCALL VTRAP(0x100, syscall_setup) 225#ifdef notyet 226#define ZS_INTERRUPT ba,a,pt %icc, zshard; nop; TA8 227#else 228#define ZS_INTERRUPT4U HARDINT4U(12) 229#endif 230 231 232/* 233 * Macro to clear %tt so we don't get confused with old traps. 234 */ 235#ifdef DEBUG 236#define CLRTT wrpr %g0,0x1ff,%tt 237#else 238#define CLRTT 239#endif 240 241/* 242 * Here are some oft repeated traps as macros. 243 */ 244 245 /* spill a 64-bit register window */ 246#define SPILL64(label,as) \ 247label: \ 248 wr %g0, as, %asi; \ 249 stxa %l0, [%sp+BIAS+0x00]%asi; \ 250 stxa %l1, [%sp+BIAS+0x08]%asi; \ 251 stxa %l2, [%sp+BIAS+0x10]%asi; \ 252 stxa %l3, [%sp+BIAS+0x18]%asi; \ 253 stxa %l4, [%sp+BIAS+0x20]%asi; \ 254 stxa %l5, [%sp+BIAS+0x28]%asi; \ 255 stxa %l6, [%sp+BIAS+0x30]%asi; \ 256 \ 257 stxa %l7, [%sp+BIAS+0x38]%asi; \ 258 stxa %i0, [%sp+BIAS+0x40]%asi; \ 259 stxa %i1, [%sp+BIAS+0x48]%asi; \ 260 stxa %i2, [%sp+BIAS+0x50]%asi; \ 261 stxa %i3, [%sp+BIAS+0x58]%asi; \ 262 stxa %i4, [%sp+BIAS+0x60]%asi; \ 263 stxa %i5, [%sp+BIAS+0x68]%asi; \ 264 stxa %i6, [%sp+BIAS+0x70]%asi; \ 265 \ 266 stxa %i7, [%sp+BIAS+0x78]%asi; \ 267 saved; \ 268 CLRTT; \ 269 retry; \ 270 NOTREACHED; \ 271 TA32 272 273 /* spill a 32-bit register window */ 274#define SPILL32(label,as) \ 275label: \ 276 wr %g0, as, %asi; \ 277 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 278 stwa %l0, [%sp+0x00]%asi; \ 279 stwa %l1, [%sp+0x04]%asi; \ 280 stwa %l2, [%sp+0x08]%asi; \ 281 stwa %l3, [%sp+0x0c]%asi; \ 282 stwa %l4, [%sp+0x10]%asi; \ 283 stwa %l5, [%sp+0x14]%asi; \ 284 \ 285 stwa %l6, [%sp+0x18]%asi; \ 286 stwa %l7, [%sp+0x1c]%asi; \ 287 stwa %i0, [%sp+0x20]%asi; \ 288 stwa %i1, [%sp+0x24]%asi; \ 289 stwa %i2, [%sp+0x28]%asi; \ 290 stwa %i3, [%sp+0x2c]%asi; \ 291 stwa %i4, [%sp+0x30]%asi; \ 292 stwa %i5, [%sp+0x34]%asi; \ 293 \ 294 stwa %i6, [%sp+0x38]%asi; \ 295 stwa %i7, [%sp+0x3c]%asi; \ 296 saved; \ 297 CLRTT; \ 298 retry; \ 299 NOTREACHED; \ 300 TA32 301 302 /* Spill either 32-bit or 64-bit register window. */ 303#define SPILLBOTH(label64,label32,as) \ 304 andcc %sp, 1, %g0; \ 305 bnz,pt %xcc, label64+4; /* Is it a v9 or v8 stack? */ \ 306 wr %g0, as, %asi; \ 307 ba,pt %xcc, label32+8; \ 308 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 309 NOTREACHED; \ 310 TA32 311 312 /* fill a 64-bit register window */ 313#define FILL64(label,as) \ 314label: \ 315 wr %g0, as, %asi; \ 316 ldxa [%sp+BIAS+0x00]%asi, %l0; \ 317 ldxa [%sp+BIAS+0x08]%asi, %l1; \ 318 ldxa [%sp+BIAS+0x10]%asi, %l2; \ 319 ldxa [%sp+BIAS+0x18]%asi, %l3; \ 320 ldxa [%sp+BIAS+0x20]%asi, %l4; \ 321 ldxa [%sp+BIAS+0x28]%asi, %l5; \ 322 ldxa [%sp+BIAS+0x30]%asi, %l6; \ 323 \ 324 ldxa [%sp+BIAS+0x38]%asi, %l7; \ 325 ldxa [%sp+BIAS+0x40]%asi, %i0; \ 326 ldxa [%sp+BIAS+0x48]%asi, %i1; \ 327 ldxa [%sp+BIAS+0x50]%asi, %i2; \ 328 ldxa [%sp+BIAS+0x58]%asi, %i3; \ 329 ldxa [%sp+BIAS+0x60]%asi, %i4; \ 330 ldxa [%sp+BIAS+0x68]%asi, %i5; \ 331 ldxa [%sp+BIAS+0x70]%asi, %i6; \ 332 \ 333 ldxa [%sp+BIAS+0x78]%asi, %i7; \ 334 restored; \ 335 CLRTT; \ 336 retry; \ 337 NOTREACHED; \ 338 TA32 339 340 /* fill a 32-bit register window */ 341#define FILL32(label,as) \ 342label: \ 343 wr %g0, as, %asi; \ 344 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 345 lda [%sp+0x00]%asi, %l0; \ 346 lda [%sp+0x04]%asi, %l1; \ 347 lda [%sp+0x08]%asi, %l2; \ 348 lda [%sp+0x0c]%asi, %l3; \ 349 lda [%sp+0x10]%asi, %l4; \ 350 lda [%sp+0x14]%asi, %l5; \ 351 \ 352 lda [%sp+0x18]%asi, %l6; \ 353 lda [%sp+0x1c]%asi, %l7; \ 354 lda [%sp+0x20]%asi, %i0; \ 355 lda [%sp+0x24]%asi, %i1; \ 356 lda [%sp+0x28]%asi, %i2; \ 357 lda [%sp+0x2c]%asi, %i3; \ 358 lda [%sp+0x30]%asi, %i4; \ 359 lda [%sp+0x34]%asi, %i5; \ 360 \ 361 lda [%sp+0x38]%asi, %i6; \ 362 lda [%sp+0x3c]%asi, %i7; \ 363 restored; \ 364 CLRTT; \ 365 retry; \ 366 NOTREACHED; \ 367 TA32 368 369 /* fill either 32-bit or 64-bit register window. */ 370#define FILLBOTH(label64,label32,as) \ 371 andcc %sp, 1, %i0; \ 372 bnz (label64)+4; /* See if it's a v9 stack or v8 */ \ 373 wr %g0, as, %asi; \ 374 ba (label32)+8; \ 375 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 376 NOTREACHED; \ 377 TA32 378 379 .globl start, _C_LABEL(kernel_text) 380 _C_LABEL(kernel_text) = kernel_start ! for kvm_mkdb(8) 381kernel_start: 382 /* Traps from TL=0 -- traps from user mode */ 383#ifdef __STDC__ 384#define TABLE(name) user_ ## name 385#else 386#define TABLE(name) user_/**/name 387#endif 388 .globl _C_LABEL(trapbase) 389_C_LABEL(trapbase): 390 b dostart; nop; TA8 ! 000 = reserved -- Use it to boot 391 /* We should not get the next 5 traps */ 392 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 393 UTRAP(0x002) ! 002 = WDR -- ROM should get this 394 UTRAP(0x003) ! 003 = XIR -- ROM should get this 395 UTRAP(0x004) ! 004 = SIR -- ROM should get this 396 UTRAP(0x005) ! 005 = RED state exception 397 UTRAP(0x006); UTRAP(0x007) 398 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 399 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss 400 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 401 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 402 TRAP(T_ILLINST) ! 010 = illegal instruction 403 TRAP(T_PRIVINST) ! 011 = privileged instruction 404 UTRAP(0x012) ! 012 = unimplemented LDD 405 UTRAP(0x013) ! 013 = unimplemented STD 406 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 407 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 408 UTRAP(0x01e); UTRAP(0x01f) 409 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 410 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 411 TRAP(T_FP_OTHER) ! 022 = other fp exception 412 TRAP(T_TAGOF) ! 023 = tag overflow 413 rdpr %cleanwin, %o7 ! 024-027 = clean window trap 414 inc %o7 ! This handler is in-lined and cannot fault 415#ifdef DEBUG 416 set 0xbadcafe, %l0 ! DEBUG -- compiler should not rely on zero-ed registers. 417#else 418 clr %l0 419#endif 420 wrpr %g0, %o7, %cleanwin ! Nucleus (trap&IRQ) code does not need clean windows 421 422 mov %l0,%l1; mov %l0,%l2 ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 423 mov %l0,%l3; mov %l0,%l4 424#if 0 425#ifdef DIAGNOSTIC 426 !! 427 !! Check the sp redzone 428 !! 429 !! Since we can't spill the current window, we'll just keep 430 !! track of the frame pointer. Problems occur when the routine 431 !! allocates and uses stack storage. 432 !! 433! rdpr %wstate, %l5 ! User stack? 434! cmp %l5, WSTATE_KERN 435! bne,pt %icc, 7f 436 sethi %hi(CPCB), %l5 437 LDPTR [%l5 + %lo(CPCB)], %l5 ! If pcb < fp < pcb+sizeof(pcb) 438 inc PCB_SIZE, %l5 ! then we have a stack overflow 439 btst %fp, 1 ! 64-bit stack? 440 sub %fp, %l5, %l7 441 bnz,a,pt %icc, 1f 442 inc BIAS, %l7 ! Remove BIAS 4431: 444 cmp %l7, PCB_SIZE 445 blu %xcc, cleanwin_overflow 446#endif 447#endif 448 mov %l0, %l5 449 mov %l0, %l6; mov %l0, %l7; mov %l0, %o0; mov %l0, %o1 450 451 mov %l0, %o2; mov %l0, %o3; mov %l0, %o4; mov %l0, %o5; 452 mov %l0, %o6; mov %l0, %o7 453 CLRTT 454 retry; nop; NOTREACHED; TA32 455 TRAP(T_DIV0) ! 028 = divide by zero 456 UTRAP(0x029) ! 029 = internal processor error 457 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 458 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 459 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 460 VTRAP(T_DATA_ERROR, winfault) ! 032 = data access error 461 VTRAP(T_DATA_PROT, winfault) ! 033 = data protection fault 462 TRAP(T_ALIGN) ! 034 = address alignment error -- we could fix it inline... 463 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 464 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 465 TRAP(T_PRIVACT) ! 037 = privileged action 466 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 467 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 468 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 469 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 470 HARDINT4U(2) ! 042 = level 2 interrupt 471 HARDINT4U(3) ! 043 = level 3 interrupt 472 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 473 HARDINT4U(5) ! 045 = level 5 interrupt 474 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 475 HARDINT4U(7) ! 047 = level 7 interrupt 476 HARDINT4U(8) ! 048 = level 8 interrupt 477 HARDINT4U(9) ! 049 = level 9 interrupt 478 HARDINT4U(10) ! 04a = level 10 interrupt 479 HARDINT4U(11) ! 04b = level 11 interrupt 480 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 481 HARDINT4U(13) ! 04d = level 13 interrupt 482 HARDINT4U(14) ! 04e = level 14 interrupt 483 HARDINT4U(15) ! 04f = nonmaskable interrupt 484 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 485 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 486 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 487 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 488 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 489 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 490 UTRAP(T_ECCERR) ! We'll implement this one later 491ufast_IMMU_miss: ! 064 = fast instr access MMU miss 492 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 493#ifdef NO_TSB 494 ba,a %icc, instr_miss 495#endif 496 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 497 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 498 brgez,pn %g5, instr_miss ! Entry invalid? Punt 499 cmp %g1, %g4 ! Compare TLB tags 500 bne,pn %xcc, instr_miss ! Got right tag? 501 nop 502 CLRTT 503 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 504 retry ! Try new mapping 5051: 506 sir 507 TA32 508ufast_DMMU_miss: ! 068 = fast data access MMU miss 509 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 510#ifdef NO_TSB 511 ba,a %icc, data_miss 512#endif 513 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 514 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 515 brgez,pn %g5, data_miss ! Entry invalid? Punt 516 cmp %g1, %g4 ! Compare TLB tags 517 bnz,pn %xcc, data_miss ! Got right tag? 518 nop 519 CLRTT 520#ifdef TRAPSTATS 521 sethi %hi(_C_LABEL(udhit)), %g1 522 lduw [%g1+%lo(_C_LABEL(udhit))], %g2 523 inc %g2 524 stw %g2, [%g1+%lo(_C_LABEL(udhit))] 525#endif 526 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 527 retry ! Try new mapping 5281: 529 sir 530 TA32 531ufast_DMMU_protection: ! 06c = fast data access MMU protection 532#ifdef TRAPSTATS 533 sethi %hi(_C_LABEL(udprot)), %g1 534 lduw [%g1+%lo(_C_LABEL(udprot))], %g2 535 inc %g2 536 stw %g2, [%g1+%lo(_C_LABEL(udprot))] 537#endif 538#ifdef HWREF 539 ba,a,pt %xcc, dmmu_write_fault 540#else 541 ba,a,pt %xcc, winfault 542#endif 543 nop 544 TA32 545 UTRAP(0x070) ! Implementation dependent traps 546 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 547 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 548 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 549TABLE(uspill): 550 SPILL64(uspill8,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode 551 SPILL32(uspill4,ASI_AIUS) ! 0x084 spill_1_normal 552 SPILLBOTH(uspill8,uspill4,ASI_AIUS) ! 0x088 spill_2_normal 553 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 554TABLE(kspill): 555 SPILL64(kspill8,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 556 SPILL32(kspill4,ASI_N) ! 0x094 spill_5_normal 557 SPILLBOTH(kspill8,kspill4,ASI_N) ! 0x098 spill_6_normal 558 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 559TABLE(uspillk): 560 SPILL64(uspillk8,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode 561 SPILL32(uspillk4,ASI_AIUS) ! 0x0a4 spill_1_other 562 SPILLBOTH(uspillk8,uspillk4,ASI_AIUS) ! 0x0a8 spill_2_other 563 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 564 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 565 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 566 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 567 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 568TABLE(ufill): 569 FILL64(ufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode 570 FILL32(ufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 571 FILLBOTH(ufill8,ufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 572 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 573TABLE(kfill): 574 FILL64(kfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode 575 FILL32(kfill4,ASI_N) ! 0x0d4 fill_5_normal 576 FILLBOTH(kfill8,kfill4,ASI_N) ! 0x0d8 fill_6_normal 577 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 578TABLE(ufillk): 579 FILL64(ufillk8,ASI_AIUS) ! 0x0e0 fill_0_other 580 FILL32(ufillk4,ASI_AIUS) ! 0x0e4 fill_1_other 581 FILLBOTH(ufillk8,ufillk4,ASI_AIUS) ! 0x0e8 fill_2_other 582 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 583 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 584 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 585 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 586 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 587TABLE(syscall): 588 SYSCALL ! 0x100 = sun syscall 589 BPT ! 0x101 = pseudo breakpoint instruction 590 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 591 SYSCALL ! 0x108 = svr4 syscall 592 SYSCALL ! 0x109 = bsd syscall 593 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 594 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 595 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 596 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 597 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 598 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 599 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 600 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 601 SYSCALL ! 0x140 SVID syscall (Solaris 2.7) 602 SYSCALL ! 0x141 SPARC International syscall 603 SYSCALL ! 0x142 OS Vendor syscall 604 SYSCALL ! 0x143 HW OEM syscall 605 STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 606 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 607 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 608 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 609 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 610 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 611 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 612 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 613 ! Traps beyond 0x17f are reserved 614 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 615 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 616 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 617 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 618 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 619 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 620 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 621 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 622 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 623 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 624 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 625 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 626 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 627 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 628 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 629 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 630 631 /* Traps from TL>0 -- traps from supervisor mode */ 632#undef TABLE 633#ifdef __STDC__ 634#define TABLE(name) nucleus_ ## name 635#else 636#define TABLE(name) nucleus_/**/name 637#endif 638trapbase_priv: 639 UTRAP(0x000) ! 000 = reserved -- Use it to boot 640 /* We should not get the next 5 traps */ 641 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 642 UTRAP(0x002) ! 002 = WDR Watchdog -- ROM should get this 643 UTRAP(0x003) ! 003 = XIR -- ROM should get this 644 UTRAP(0x004) ! 004 = SIR -- ROM should get this 645 UTRAP(0x005) ! 005 = RED state exception 646 UTRAP(0x006); UTRAP(0x007) 647ktextfault: 648 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 649 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss -- no MMU 650 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 651 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 652 TRAP(T_ILLINST) ! 010 = illegal instruction 653 TRAP(T_PRIVINST) ! 011 = privileged instruction 654 UTRAP(0x012) ! 012 = unimplemented LDD 655 UTRAP(0x013) ! 013 = unimplemented STD 656 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 657 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 658 UTRAP(0x01e); UTRAP(0x01f) 659 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 660 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 661 TRAP(T_FP_OTHER) ! 022 = other fp exception 662 TRAP(T_TAGOF) ! 023 = tag overflow 663 clr %l0 664#ifdef DEBUG 665 set 0xbadbeef, %l0 ! DEBUG 666#endif 667 mov %l0, %l1; mov %l0, %l2 ! 024-027 = clean window trap 668 rdpr %cleanwin, %o7 ! This handler is in-lined and cannot fault 669 inc %o7; mov %l0, %l3 ! Nucleus (trap&IRQ) code does not need clean windows 670 wrpr %g0, %o7, %cleanwin ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 671#ifdef NOT_DEBUG 672 !! 673 !! Check the sp redzone 674 !! 675 rdpr %wstate, t1 676 cmp t1, WSTATE_KERN 677 bne,pt icc, 7f 678 sethi %hi(_C_LABEL(redzone)), t1 679 ldx [t1 + %lo(_C_LABEL(redzone))], t2 680 cmp %sp, t2 ! if sp >= t2, not in red zone 681 blu panic_red ! and can continue normally 6827: 683#endif 684 mov %l0, %l4; mov %l0, %l5; mov %l0, %l6; mov %l0, %l7 685 mov %l0, %o0; mov %l0, %o1; mov %l0, %o2; mov %l0, %o3 686 687 mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7 688 CLRTT 689 retry; nop; TA32 690 TRAP(T_DIV0) ! 028 = divide by zero 691 UTRAP(0x029) ! 029 = internal processor error 692 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 693kdatafault: 694 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 695 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 696 VTRAP(T_DATA_ERROR, winfault) ! 032 = data fetch fault 697 VTRAP(T_DATA_PROT, winfault) ! 033 = data fetch fault 698 VTRAP(T_ALIGN, checkalign) ! 034 = address alignment error -- we could fix it inline... 699 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 700 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 701 TRAP(T_PRIVACT) ! 037 = privileged action 702 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 703 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 704 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 705 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 706 HARDINT4U(2) ! 042 = level 2 interrupt 707 HARDINT4U(3) ! 043 = level 3 interrupt 708 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 709 HARDINT4U(5) ! 045 = level 5 interrupt 710 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 711 HARDINT4U(7) ! 047 = level 7 interrupt 712 HARDINT4U(8) ! 048 = level 8 interrupt 713 HARDINT4U(9) ! 049 = level 9 interrupt 714 HARDINT4U(10) ! 04a = level 10 interrupt 715 HARDINT4U(11) ! 04b = level 11 interrupt 716 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 717 HARDINT4U(13) ! 04d = level 13 interrupt 718 HARDINT4U(14) ! 04e = level 14 interrupt 719 HARDINT4U(15) ! 04f = nonmaskable interrupt 720 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 721 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 722 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 723 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 724 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 725 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 726 UTRAP(T_ECCERR) ! We'll implement this one later 727kfast_IMMU_miss: ! 064 = fast instr access MMU miss 728 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 729#ifdef NO_TSB 730 ba,a %icc, instr_miss 731#endif 732 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 733 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 734 brgez,pn %g5, instr_miss ! Entry invalid? Punt 735 cmp %g1, %g4 ! Compare TLB tags 736 bne,pn %xcc, instr_miss ! Got right tag? 737 nop 738 CLRTT 739 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 740 retry ! Try new mapping 7411: 742 sir 743 TA32 744kfast_DMMU_miss: ! 068 = fast data access MMU miss 745 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 746#ifdef NO_TSB 747 ba,a %icc, data_miss 748#endif 749 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 750 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 751 brgez,pn %g5, data_miss ! Entry invalid? Punt 752 cmp %g1, %g4 ! Compare TLB tags 753 bnz,pn %xcc, data_miss ! Got right tag? 754 nop 755 CLRTT 756#ifdef TRAPSTATS 757 sethi %hi(_C_LABEL(kdhit)), %g1 758 lduw [%g1+%lo(_C_LABEL(kdhit))], %g2 759 inc %g2 760 stw %g2, [%g1+%lo(_C_LABEL(kdhit))] 761#endif 762 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 763 retry ! Try new mapping 7641: 765 sir 766 TA32 767kfast_DMMU_protection: ! 06c = fast data access MMU protection 768#ifdef TRAPSTATS 769 sethi %hi(_C_LABEL(kdprot)), %g1 770 lduw [%g1+%lo(_C_LABEL(kdprot))], %g2 771 inc %g2 772 stw %g2, [%g1+%lo(_C_LABEL(kdprot))] 773#endif 774#ifdef HWREF 775 ba,a,pt %xcc, dmmu_write_fault 776#else 777 ba,a,pt %xcc, winfault 778#endif 779 nop 780 TA32 781 UTRAP(0x070) ! Implementation dependent traps 782 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 783 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 784 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 785TABLE(uspill): 786 SPILL64(1,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows 787 SPILL32(2,ASI_AIUS) ! 0x084 spill_1_normal 788 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x088 spill_2_normal 789 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 790TABLE(kspill): 791 SPILL64(1,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 792 SPILL32(2,ASI_N) ! 0x094 spill_5_normal 793 SPILLBOTH(1b,2b,ASI_N) ! 0x098 spill_6_normal 794 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 795TABLE(uspillk): 796 SPILL64(1,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in nucleus mode 797 SPILL32(2,ASI_AIUS) ! 0x0a4 spill_1_other 798 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x0a8 spill_2_other 799 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 800 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 801 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 802 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 803 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 804TABLE(ufill): 805 FILL64(nufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running nucleus mode from user 806 FILL32(nufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 807 FILLBOTH(nufill8,nufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 808 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 809TABLE(sfill): 810 FILL64(sfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running nucleus mode from supervisor 811 FILL32(sfill4,ASI_N) ! 0x0d4 fill_5_normal 812 FILLBOTH(sfill8,sfill4,ASI_N) ! 0x0d8 fill_6_normal 813 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 814TABLE(kfill): 815 FILL64(nkfill8,ASI_AIUS) ! 0x0e0 fill_0_other -- used to fill user windows when running nucleus mode -- will we ever use this? 816 FILL32(nkfill4,ASI_AIUS) ! 0x0e4 fill_1_other 817 FILLBOTH(nkfill8,nkfill4,ASI_AIUS)! 0x0e8 fill_2_other 818 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 819 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 820 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 821 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 822 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 823TABLE(syscall): 824 SYSCALL ! 0x100 = sun syscall 825 BPT ! 0x101 = pseudo breakpoint instruction 826 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 827 SYSCALL ! 0x108 = svr4 syscall 828 SYSCALL ! 0x109 = bsd syscall 829 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 830 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 831 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 832 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 833 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 834 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 835 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 836 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 837 STRAP(0x140); STRAP(0x141); STRAP(0x142); STRAP(0x143); STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 838 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 839 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 840 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 841 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 842 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 843 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 844 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 845 ! Traps beyond 0x17f are reserved 846 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 847 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 848 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 849 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 850 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 851 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 852 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 853 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 854 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 855 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 856 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 857 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 858 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 859 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 860 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 861 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 862 863#if 0 864/* 865 * If the cleanwin trap handler detects an overfow we come here. 866 * We need to fix up the window registers, switch to the interrupt 867 * stack, and then trap to the debugger. 868 */ 869cleanwin_overflow: 870 !! We've already incremented %cleanwin 871 !! So restore %cwp 872 rdpr %cwp, %l0 873 dec %l0 874 wrpr %l0, %g0, %cwp 875 set EINTSTACK-STKB-CC64FSZ, %l0 876 save %l0, 0, %sp 877 878 ta 1 ! Enter debugger 879 sethi %hi(1f), %o0 880 call _C_LABEL(panic) 881 or %o0, %lo(1f), %o0 882 restore 883 retry 884 .data 8851: 886 .asciz "Kernel stack overflow!" 887 _ALIGN 888 .text 889#endif 890 891#ifdef NOTDEF_DEBUG 892/* 893 * A hardware red zone is impossible. We simulate one in software by 894 * keeping a `red zone' pointer; if %sp becomes less than this, we panic. 895 * This is expensive and is only enabled when debugging. 896 */ 897#define REDSIZE (PCB_SIZE) /* Mark used portion of pcb structure out of bounds */ 898#define REDSTACK 2048 /* size of `panic: stack overflow' region */ 899 .data 900 _ALIGN 901redzone: 902 .xword _C_LABEL(XXX) + REDSIZE 903redstack: 904 .space REDSTACK 905eredstack: 906Lpanic_red: 907 .asciz "kernel stack overflow" 908 _ALIGN 909 .text 910 911 /* set stack pointer redzone to base+minstack; alters base */ 912#define SET_SP_REDZONE(base, tmp) \ 913 add base, REDSIZE, base; \ 914 sethi %hi(_C_LABEL(redzone)), tmp; \ 915 stx base, [tmp + %lo(_C_LABEL(redzone))] 916 917 /* variant with a constant */ 918#define SET_SP_REDZONE_CONST(const, tmp1, tmp2) \ 919 set (const) + REDSIZE, tmp1; \ 920 sethi %hi(_C_LABEL(redzone)), tmp2; \ 921 stx tmp1, [tmp2 + %lo(_C_LABEL(redzone))] 922 923 /* check stack pointer against redzone (uses two temps) */ 924#define CHECK_SP_REDZONE(t1, t2) \ 925 sethi KERNBASE, t1; \ 926 cmp %sp, t1; \ 927 blu,pt %xcc, 7f; \ 928 sethi %hi(_C_LABEL(redzone)), t1; \ 929 ldx [t1 + %lo(_C_LABEL(redzone))], t2; \ 930 cmp %sp, t2; /* if sp >= t2, not in red zone */ \ 931 blu panic_red; nop; /* and can continue normally */ \ 9327: 933 934panic_red: 935 /* move to panic stack */ 936 stx %g0, [t1 + %lo(_C_LABEL(redzone))]; 937 set eredstack - BIAS, %sp; 938 /* prevent panic() from lowering ipl */ 939 sethi %hi(_C_LABEL(panicstr)), t2; 940 set Lpanic_red, t2; 941 st t2, [t1 + %lo(_C_LABEL(panicstr))]; 942 wrpr g0, 15, %pil /* t1 = splhigh() */ 943 save %sp, -CCF64SZ, %sp; /* preserve current window */ 944 sethi %hi(Lpanic_red), %o0; 945 call _C_LABEL(panic); 946 or %o0, %lo(Lpanic_red), %o0; 947 948 949#else 950 951#define SET_SP_REDZONE(base, tmp) 952#define SET_SP_REDZONE_CONST(const, t1, t2) 953#define CHECK_SP_REDZONE(t1, t2) 954#endif 955 956#define TRACESIZ 0x01000 957 .globl _C_LABEL(trap_trace) 958 .globl _C_LABEL(trap_trace_ptr) 959 .globl _C_LABEL(trap_trace_end) 960 .globl _C_LABEL(trap_trace_dis) 961 .data 962_C_LABEL(trap_trace_dis): 963 .word 1, 1 ! Starts disabled. DDB turns it on. 964_C_LABEL(trap_trace_ptr): 965 .word 0, 0, 0, 0 966_C_LABEL(trap_trace): 967 .space TRACESIZ 968_C_LABEL(trap_trace_end): 969 .space 0x20 ! safety margin 970 971 972/* 973 * v9 machines do not have a trap window. 974 * 975 * When we take a trap the trap state is pushed on to the stack of trap 976 * registers, interrupts are disabled, then we switch to an alternate set 977 * of global registers. 978 * 979 * The trap handling code needs to allocate a trap frame on the kernel, or 980 * for interrupts, the interrupt stack, save the out registers to the trap 981 * frame, then switch to the normal globals and save them to the trap frame 982 * too. 983 * 984 * XXX it would be good to save the interrupt stack frame to the kernel 985 * stack so we wouldn't have to copy it later if we needed to handle a AST. 986 * 987 * Since kernel stacks are all on one page and the interrupt stack is entirely 988 * within the locked TLB, we can use physical addressing to save out our 989 * trap frame so we don't trap during the TRAP_SETUP() operation. There 990 * is unfortunately no supportable method for issuing a non-trapping save. 991 * 992 * However, if we use physical addresses to save our trapframe, we will need 993 * to clear out the data cache before continuing much further. 994 * 995 * In short, what we need to do is: 996 * 997 * all preliminary processing is done using the alternate globals 998 * 999 * When we allocate our trap windows we must give up our globals because 1000 * their state may have changed during the save operation 1001 * 1002 * we need to save our normal globals as soon as we have a stack 1003 * 1004 * Finally, we may now call C code. 1005 * 1006 * This macro will destroy %g5-%g7. %g0-%g4 remain unchanged. 1007 * 1008 * In order to properly handle nested traps without lossage, alternate 1009 * global %g6 is used as a kernel stack pointer. It is set to the last 1010 * allocated stack pointer (trapframe) and the old value is stored in 1011 * tf_kstack. It is restored when returning from a trap. It is cleared 1012 * on entering user mode. 1013 */ 1014 1015 /* 1016 * Other misc. design criteria: 1017 * 1018 * When taking an address fault, fault info is in the sfsr, sfar, 1019 * TLB_TAG_ACCESS registers. If we take another address fault 1020 * while trying to handle the first fault then that information, 1021 * the only information that tells us what address we trapped on, 1022 * can potentially be lost. This trap can be caused when allocating 1023 * a register window with which to handle the trap because the save 1024 * may try to store or restore a register window that corresponds 1025 * to part of the stack that is not mapped. Preventing this trap, 1026 * while possible, is much too complicated to do in a trap handler, 1027 * and then we will need to do just as much work to restore the processor 1028 * window state. 1029 * 1030 * Possible solutions to the problem: 1031 * 1032 * Since we have separate AG, MG, and IG, we could have all traps 1033 * above level-1 preserve AG and use other registers. This causes 1034 * a problem for the return from trap code which is coded to use 1035 * alternate globals only. 1036 * 1037 * We could store the trapframe and trap address info to the stack 1038 * using physical addresses. Then we need to read it back using 1039 * physical addressing, or flush the D$. 1040 * 1041 * We could identify certain registers to hold address fault info. 1042 * this means that these registers need to be preserved across all 1043 * fault handling. But since we only have 7 useable globals, that 1044 * really puts a cramp in our style. 1045 * 1046 * Finally, there is the issue of returning from kernel mode to user 1047 * mode. If we need to issue a restore of a user window in kernel 1048 * mode, we need the window control registers in a user mode setup. 1049 * If the trap handlers notice the register windows are in user mode, 1050 * they will allocate a trapframe at the bottom of the kernel stack, 1051 * overwriting the frame we were trying to return to. This means that 1052 * we must complete the restoration of all registers *before* switching 1053 * to a user-mode window configuration. 1054 * 1055 * Essentially we need to be able to write re-entrant code w/no stack. 1056 */ 1057 .data 1058trap_setup_msg: 1059 .asciz "TRAP_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1060 _ALIGN 1061intr_setup_msg: 1062 .asciz "INTR_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1063 _ALIGN 1064 .text 1065 1066#ifdef DEBUG 1067 /* Only save a snapshot of locals and ins in DEBUG kernels */ 1068#define SAVE_LOCALS_INS \ 1069 /* Save local registers to trap frame */ \ 1070 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)]; \ 1071 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)]; \ 1072 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)]; \ 1073 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)]; \ 1074 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)]; \ 1075 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)]; \ 1076 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)]; \ 1077 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)]; \ 1078\ 1079 /* Save in registers to trap frame */ \ 1080 stx %i0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)]; \ 1081 stx %i1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)]; \ 1082 stx %i2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)]; \ 1083 stx %i3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)]; \ 1084 stx %i4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)]; \ 1085 stx %i5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)]; \ 1086 stx %i6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)]; \ 1087 stx %i7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)]; \ 1088\ 1089 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT]; 1090#else 1091#define SAVE_LOCALS_INS 1092#endif 1093 1094#ifdef _LP64 1095#define FIXUP_TRAP_STACK \ 1096 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1097 bnz,pt %icc, 1f; \ 1098 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1099 inc -BIAS, %g6; \ 11001: 1101#else 1102#define FIXUP_TRAP_STACK \ 1103 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1104 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1105 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1106 add %g6, BIAS, %g5; \ 1107 movne %icc, %g5, %g6; 1108#endif 1109 1110#ifdef _LP64 1111#define TRAP_SETUP(stackspace) \ 1112 sethi %hi(CPCB), %g6; \ 1113 sethi %hi((stackspace)), %g5; \ 1114 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1115 sethi %hi(USPACE), %g7; /* Always multiple of page size */ \ 1116 or %g5, %lo((stackspace)), %g5; \ 1117 add %g6, %g7, %g6; \ 1118 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1119 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1120 \ 1121 sub %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1122 movrz %g7, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1123 FIXUP_TRAP_STACK \ 1124 SAVE_LOCALS_INS \ 1125 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1126 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1127 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1128 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1129 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1130 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1131 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1132\ 1133 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1134 brz,pt %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1135 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1136 mov CTX_PRIMARY, %g7; \ 1137 /* came from user mode -- switch to kernel mode stack */ \ 1138 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1139 wrpr %g0, 0, %canrestore; \ 1140 wrpr %g0, %g5, %otherwin; \ 1141 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1142\ 1143 stxa %g0, [%g7] ASI_DMMU; /* Switch MMU to kernel primary context */ \ 1144 sethi %hi(KERNBASE), %g5; \ 1145 flush %g5; /* Some convenient address that won't trap */ \ 11461: 1147 1148/* 1149 * Interrupt setup is almost exactly like trap setup, but we need to 1150 * go to the interrupt stack if (a) we came from user mode or (b) we 1151 * came from kernel mode on the kernel stack. 1152 * 1153 * We don't guarantee any registers are preserved during this operation. 1154 * So we can be more efficient. 1155 */ 1156#define INTR_SETUP(stackspace) \ 1157 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1158 \ 1159 sethi %hi(EINTSTACK-BIAS), %g6; \ 1160 sethi %hi(EINTSTACK-INTSTACK), %g4; \ 1161 \ 1162 or %g6, %lo(EINTSTACK-BIAS), %g6; /* Base of interrupt stack */ \ 1163 dec %g4; /* Make it into a mask */ \ 1164 \ 1165 sub %g6, %sp, %g1; /* Offset from interrupt stack */ \ 1166 sethi %hi((stackspace)), %g5; \ 1167 \ 1168 or %g5, %lo((stackspace)), %g5; \ 1169\ 1170 andn %g1, %g4, %g4; /* Are we out of the interrupt stack range? */ \ 1171 xor %g7, WSTATE_KERN, %g3; /* Are we on the user stack ? */ \ 1172 \ 1173 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1174 or %g3, %g4, %g4; /* Definitely not off the interrupt stack */ \ 1175 \ 1176 movrz %g4, %sp, %g6; \ 1177 \ 1178 add %g6, %g5, %g5; /* Allocate a stack frame */ \ 1179 btst 1, %g6; \ 1180 bnz,pt %icc, 1f; \ 1181\ 1182 mov %g5, %g6; \ 1183 \ 1184 add %g5, -BIAS, %g6; \ 1185 \ 11861: SAVE_LOCALS_INS \ 1187 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1188 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1189 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1190 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1191 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1192 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1193\ 1194 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1195 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1196 stx %i6, [%sp + CC64FSZ + BIAS + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1197 brz,pt %g3, 1f; /* If we were in kernel mode start saving globals */ \ 1198 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1199 /* came from user mode -- switch to kernel mode stack */ \ 1200 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1201 \ 1202 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1203 \ 1204 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1205\ 1206 wrpr %g0, 0, %canrestore; \ 1207 \ 1208 wrpr %g0, %g5, %otherwin; \ 1209 \ 1210 sethi %hi(KERNBASE), %g5; \ 1211 mov CTX_PRIMARY, %g7; \ 1212 \ 1213 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1214 \ 1215 stxa %g0, [%g7] ASI_DMMU; /* Switch MMU to kernel primary context */ \ 1216 \ 1217 flush %g5; /* Some convenient address that won't trap */ \ 12181: 1219 1220#else /* _LP64 */ 1221 1222#define TRAP_SETUP(stackspace) \ 1223 sethi %hi(CPCB), %g6; \ 1224 sethi %hi((stackspace)), %g5; \ 1225 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1226 sethi %hi(USPACE), %g7; \ 1227 or %g5, %lo((stackspace)), %g5; \ 1228 add %g6, %g7, %g6; \ 1229 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1230 \ 1231 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1232 subcc %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1233 movz %icc, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1234 FIXUP_TRAP_STACK \ 1235 SAVE_LOCALS_INS \ 1236 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1237 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1238 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1239 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1240 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1241 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1242 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1243 \ 1244 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1245 brz,pn %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1246 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1247 mov CTX_PRIMARY, %g7; \ 1248 /* came from user mode -- switch to kernel mode stack */ \ 1249 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1250 wrpr %g0, 0, %canrestore; \ 1251 wrpr %g0, %g5, %otherwin; \ 1252 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1253 \ 1254 stxa %g0, [%g7] ASI_DMMU; /* Switch MMU to kernel primary context */ \ 1255 sethi %hi(KERNBASE), %g5; \ 1256 flush %g5; /* Some convenient address that won't trap */ \ 12571: 1258 1259/* 1260 * Interrupt setup is almost exactly like trap setup, but we need to 1261 * go to the interrupt stack if (a) we came from user mode or (b) we 1262 * came from kernel mode on the kernel stack. 1263 * 1264 * We don't guarantee any registers are preserved during this operation. 1265 */ 1266#define INTR_SETUP(stackspace) \ 1267 sethi %hi(EINTSTACK), %g1; \ 1268 sethi %hi((stackspace)), %g5; \ 1269 btst 1, %sp; \ 1270 add %sp, BIAS, %g6; \ 1271 movz %icc, %sp, %g6; \ 1272 or %g1, %lo(EINTSTACK), %g1; \ 1273 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1274 set (EINTSTACK-INTSTACK), %g7; \ 1275 or %g5, %lo((stackspace)), %g5; \ 1276 sub %g1, %g6, %g2; /* Determine if we need to switch to intr stack or not */ \ 1277 dec %g7; /* Make it into a mask */ \ 1278 andncc %g2, %g7, %g0; /* XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 1279 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1280 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1281 movnz %xcc, %g1, %g6; /* Stay on interrupt stack? */ \ 1282 cmp %g7, WSTATE_KERN; /* User or kernel sp? */ \ 1283 movnz %icc, %g1, %g6; /* Stay on interrupt stack? */ \ 1284 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1285 \ 1286 SAVE_LOCALS_INS \ 1287 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1288 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1289 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1290 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1291 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1292 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1293 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1294 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1295 stx %i6, [%sp + CC64FSZ + STKB + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1296 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1297 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1298 cmp %g7, WSTATE_KERN; /* Compare & leave in register */ \ 1299 be,pn %icc, 1f; /* If we were in kernel mode start saving globals */ \ 1300 /* came from user mode -- switch to kernel mode stack */ \ 1301 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1302 tst %g5; tnz %xcc, 1; nop; /* DEBUG -- this should _NEVER_ happen */ \ 1303 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1304 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1305 wrpr %g0, 0, %canrestore; \ 1306 mov CTX_PRIMARY, %g7; \ 1307 wrpr %g0, %g5, %otherwin; \ 1308 sethi %hi(KERNBASE), %g5; \ 1309 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1310 stxa %g0, [%g7] ASI_DMMU; /* Switch MMU to kernel primary context */ \ 1311 flush %g5; /* Some convenient address that won't trap */ \ 13121: 1313#endif /* _LP64 */ 1314 1315#ifdef DEBUG 1316 1317 /* Look up kpte to test algorithm */ 1318 .globl asmptechk 1319asmptechk: 1320 mov %o0, %g4 ! pmap->pm_segs 1321 mov %o1, %g3 ! Addr to lookup -- mind the context 1322 1323 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1324 brz,pt %g5, 0f ! Should be zero or -1 1325 inc %g5 ! Make -1 -> 0 1326 brnz,pn %g5, 1f ! Error! 13270: 1328 srlx %g3, STSHIFT, %g5 1329 and %g5, STMASK, %g5 1330 sll %g5, 3, %g5 1331 add %g4, %g5, %g4 1332 DLFLUSH(%g4,%g5) 1333 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1334 DLFLUSH2(%g5) 1335 brz,pn %g4, 1f ! NULL entry? check somewhere else 1336 1337 srlx %g3, PDSHIFT, %g5 1338 and %g5, PDMASK, %g5 1339 sll %g5, 3, %g5 1340 add %g4, %g5, %g4 1341 DLFLUSH(%g4,%g5) 1342 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1343 DLFLUSH2(%g5) 1344 brz,pn %g4, 1f ! NULL entry? check somewhere else 1345 1346 srlx %g3, PTSHIFT, %g5 ! Convert to ptab offset 1347 and %g5, PTMASK, %g5 1348 sll %g5, 3, %g5 1349 add %g4, %g5, %g4 1350 DLFLUSH(%g4,%g5) 1351 ldxa [%g4] ASI_PHYS_CACHED, %g6 1352 DLFLUSH2(%g5) 1353 brgez,pn %g6, 1f ! Entry invalid? Punt 1354 srlx %g6, 32, %o0 1355 retl 1356 srl %g6, 0, %o1 13571: 1358 mov %g0, %o1 1359 retl 1360 mov %g0, %o0 1361 1362 .data 13632: 1364 .asciz "asmptechk: %x %x %x %x:%x\r\n" 1365 _ALIGN 1366 .text 1367#endif 1368 1369/* 1370 * This is the MMU protection handler. It's too big to fit 1371 * in the trap table so I moved it here. It's relatively simple. 1372 * It looks up the page mapping in the page table associated with 1373 * the trapping context. It checks to see if the S/W writable bit 1374 * is set. If so, it sets the H/W write bit, marks the tte modified, 1375 * and enters the mapping into the MMU. Otherwise it does a regular 1376 * data fault. 1377 */ 1378 ICACHE_ALIGN 1379dmmu_write_fault: 1380 mov TLB_TAG_ACCESS, %g3 1381 sethi %hi(0x1fff), %g6 ! 8K context mask 1382 ldxa [%g3] ASI_DMMU, %g3 ! Get fault addr from Tag Target 1383 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1384 or %g6, %lo(0x1fff), %g6 1385 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1386 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1387 and %g3, %g6, %g6 ! Isolate context 1388 1389 inc %g5 ! (0 or -1) -> (1 or 0) 1390 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1391 ldx [%g4+%g6], %g4 ! Load up our page table. 1392 srlx %g3, STSHIFT, %g6 1393 cmp %g5, 1 1394 bgu,pn %xcc, winfix ! Error! 1395 srlx %g3, PDSHIFT, %g5 1396 and %g6, STMASK, %g6 1397 sll %g6, 3, %g6 1398 1399 and %g5, PDMASK, %g5 1400 sll %g5, 3, %g5 1401 add %g6, %g4, %g4 1402 DLFLUSH(%g4,%g6) 1403 ldxa [%g4] ASI_PHYS_CACHED, %g4 1404 DLFLUSH2(%g6) 1405 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1406 and %g6, PTMASK, %g6 1407 add %g5, %g4, %g5 1408 brz,pn %g4, winfix ! NULL entry? check somewhere else 1409 nop 1410 1411 ldxa [%g5] ASI_PHYS_CACHED, %g4 1412 sll %g6, 3, %g6 1413 brz,pn %g4, winfix ! NULL entry? check somewhere else 1414 add %g6, %g4, %g6 14151: 1416 ldxa [%g6] ASI_PHYS_CACHED, %g4 1417 brgez,pn %g4, winfix ! Entry invalid? Punt 1418 or %g4, TTE_MODIFY|TTE_ACCESS|TTE_W, %g7 ! Update the modified bit 1419 1420 btst TTE_REAL_W|TTE_W, %g4 ! Is it a ref fault? 1421 bz,pn %xcc, winfix ! No -- really fault 1422#ifdef DEBUG 1423 /* Make sure we don't try to replace a kernel translation */ 1424 /* This should not be necessary */ 1425 sllx %g3, 64-13, %g2 ! Isolate context bits 1426 sethi %hi(KERNBASE), %g5 ! Don't need %lo 1427 brnz,pt %g2, 0f ! Ignore context != 0 1428 set 0x0800000, %g2 ! 8MB 1429 sub %g3, %g5, %g5 1430 cmp %g5, %g2 1431 tlu %xcc, 1; nop 1432 blu,pn %xcc, winfix ! Next insn in delay slot is unimportant 14330: 1434#endif 1435 /* Need to check for and handle large pages. */ 1436 srlx %g4, 61, %g5 ! Isolate the size bits 1437 ldxa [%g0] ASI_DMMU_8KPTR, %g2 ! Load DMMU 8K TSB pointer 1438 andcc %g5, 0x3, %g5 ! 8K? 1439 bnz,pn %icc, winfix ! We punt to the pmap code since we can't handle policy 1440 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 1441 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1442 membar #StoreLoad 1443 cmp %g4, %g7 1444 bne,pn %xcc, 1b 1445 or %g4, TTE_MODIFY|TTE_ACCESS|TTE_W, %g4 ! Update the modified bit 1446 stx %g1, [%g2] ! Update TSB entry tag 1447 mov SFSR, %g7 1448 stx %g4, [%g2+8] ! Update TSB entry data 1449 nop 1450 1451#ifdef TRAPSTATS 1452 sethi %hi(_C_LABEL(protfix)), %g1 1453 lduw [%g1+%lo(_C_LABEL(protfix))], %g2 1454 inc %g2 1455 stw %g2, [%g1+%lo(_C_LABEL(protfix))] 1456#endif 1457 mov DEMAP_PAGE_SECONDARY, %g1 ! Secondary flush 1458 mov DEMAP_PAGE_NUCLEUS, %g5 ! Nucleus flush 1459 stxa %g0, [%g7] ASI_DMMU ! clear out the fault 1460 sllx %g3, (64-13), %g7 ! Need to demap old entry first 1461 andn %g3, 0xfff, %g6 1462 movrz %g7, %g5, %g1 ! Pick one 1463 or %g6, %g1, %g6 1464 membar #Sync 1465 stxa %g6, [%g6] ASI_DMMU_DEMAP ! Do the demap 1466 membar #Sync 1467 1468 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1469 membar #Sync 1470 retry 1471 1472/* 1473 * Each memory data access fault from a fast access miss handler comes here. 1474 * We will quickly check if this is an original prom mapping before going 1475 * to the generic fault handler 1476 * 1477 * We will assume that %pil is not lost so we won't bother to save it 1478 * unless we're in an interrupt handler. 1479 * 1480 * On entry: 1481 * We are on one of the alternate set of globals 1482 * %g1 = MMU tag target 1483 * %g2 = 8Kptr 1484 * %g3 = TLB TAG ACCESS 1485 * 1486 * On return: 1487 * 1488 */ 1489 ICACHE_ALIGN 1490data_miss: 1491#ifdef TRAPSTATS 1492 set _C_LABEL(kdmiss), %g3 1493 set _C_LABEL(udmiss), %g4 1494 rdpr %tl, %g6 1495 dec %g6 1496 movrz %g6, %g4, %g3 1497 lduw [%g3], %g4 1498 inc %g4 1499 stw %g4, [%g3] 1500#endif 1501 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 1502 sethi %hi(0x1fff), %g6 ! 8K context mask 1503 ldxa [%g3] ASI_DMMU, %g3 ! from tag access register 1504 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1505 or %g6, %lo(0x1fff), %g6 1506 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1507 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1508 and %g3, %g6, %g6 ! Isolate context 1509 1510 inc %g5 ! (0 or -1) -> (1 or 0) 1511 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1512 ldx [%g4+%g6], %g4 ! Load up our page table. 1513#ifdef DEBUG 1514 /* Make sure we don't try to replace a kernel translation */ 1515 /* This should not be necessary */ 1516 brnz,pt %g6, 1f ! If user context continue miss 1517 sethi %hi(KERNBASE), %g7 ! Don't need %lo 1518 set 0x0800000, %g6 ! 8MB 1519 sub %g3, %g7, %g7 1520 cmp %g7, %g6 1521 tlu %xcc, 1; nop 15221: 1523#endif 1524 srlx %g3, STSHIFT, %g6 1525 cmp %g5, 1 1526 bgu,pn %xcc, winfix ! Error! 1527 srlx %g3, PDSHIFT, %g5 1528 and %g6, STMASK, %g6 1529 1530 sll %g6, 3, %g6 1531 and %g5, PDMASK, %g5 1532 sll %g5, 3, %g5 1533 add %g6, %g4, %g4 1534 ldxa [%g4] ASI_PHYS_CACHED, %g4 1535 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1536 and %g6, PTMASK, %g6 1537 add %g5, %g4, %g5 1538 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1539 1540 nop 1541 ldxa [%g5] ASI_PHYS_CACHED, %g4 1542 sll %g6, 3, %g6 1543 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1544 add %g6, %g4, %g6 1545 15461: 1547 ldxa [%g6] ASI_PHYS_CACHED, %g4 1548 brgez,pn %g4, data_nfo ! Entry invalid? Punt 1549 or %g4, TTE_ACCESS, %g7 ! Update the access bit 1550 1551 btst TTE_ACCESS, %g4 ! Need to update access git? 1552 bne,pt %xcc, 1f 1553 nop 1554 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1555 cmp %g4, %g7 1556 bne,pn %xcc, 1b 1557 or %g4, TTE_ACCESS, %g4 ! Update the access bit 1558 15591: 1560 stx %g1, [%g2] ! Update TSB entry tag 1561 stx %g4, [%g2+8] ! Update TSB entry data 1562 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1563 membar #Sync 1564 CLRTT 1565 retry 1566 NOTREACHED 1567/* 1568 * We had a data miss but did not find a mapping. Insert 1569 * a NFO mapping to satisfy speculative loads and return. 1570 * If this had been a real load, it will re-execute and 1571 * result in a data fault or protection fault rather than 1572 * a TLB miss. We insert an 8K TTE with the valid and NFO 1573 * bits set. All others should zero. The TTE looks like this: 1574 * 1575 * 0x9000000000000000 1576 * 1577 */ 1578data_nfo: 1579 sethi %hi(0x90000000), %g4 ! V(0x8)|NFO(0x1) 1580 sllx %g4, 32, %g4 1581 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1582 membar #Sync 1583 CLRTT 1584 retry 1585 1586/* 1587 * Handler for making the trap window shiny clean. 1588 * 1589 * If the store that trapped was to a kernel address, panic. 1590 * 1591 * If the store that trapped was to a user address, stick it in the PCB. 1592 * Since we don't want to force user code to use the standard register 1593 * convention if we don't have to, we will not assume that %fp points to 1594 * anything valid. 1595 * 1596 * On entry: 1597 * We are on one of the alternate set of globals 1598 * %g1 = %tl - 1, tstate[tl-1], scratch - local 1599 * %g2 = %tl - local 1600 * %g3 = MMU tag access - in 1601 * %g4 = %cwp - local 1602 * %g5 = scratch - local 1603 * %g6 = cpcb - local 1604 * %g7 = scratch - local 1605 * 1606 * On return: 1607 * 1608 * NB: remove most of this from main codepath & cleanup I$ 1609 */ 1610winfault: 1611 mov TLB_TAG_ACCESS, %g3 ! Get real fault page from tag access register 1612 ldxa [%g3] ASI_DMMU, %g3 ! And put it into the non-MMU alternate regs 1613winfix: 1614 rdpr %tl, %g2 1615 subcc %g2, 1, %g1 1616 ble,pt %icc, datafault ! Don't go below trap level 1 1617 sethi %hi(CPCB), %g6 ! get current pcb 1618 1619 1620 wrpr %g1, 0, %tl ! Pop a trap level 1621 rdpr %tt, %g7 ! Read type of prev. trap 1622 rdpr %tstate, %g4 ! Try to restore prev %cwp if we were executing a restore 1623 andn %g7, 0x3f, %g5 ! window fill traps are all 0b 0000 11xx xxxx 1624 1625#if 1 1626 cmp %g7, 0x30 ! If we took a datafault just before this trap 1627 bne,pt %icc, winfixfill ! our stack's probably bad so we need to switch somewhere else 1628 nop 1629 1630 !! 1631 !! Double data fault -- bad stack? 1632 !! 1633 wrpr %g2, %tl ! Restore trap level. 1634 sir ! Just issue a reset and don't try to recover. 1635 mov %fp, %l6 ! Save the frame pointer 1636 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 1637 add %fp, -CC64FSZ, %sp ! Create a stackframe 1638 wrpr %g0, 15, %pil ! Disable interrupts, too 1639 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 1640 wrpr %g0, 7, %cansave ! probably is too, so blow away 1641 ba slowtrap ! all our register windows. 1642 wrpr %g0, 0x101, %tt 1643#endif 1644 1645winfixfill: 1646 cmp %g5, 0x0c0 ! so we mask lower bits & compare to 0b 0000 1100 0000 1647 bne,pt %icc, winfixspill ! Dump our trap frame -- we will retry the fill when the page is loaded 1648 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 1649 1650 !! 1651 !! This was a fill 1652 !! 1653#ifdef TRAPSTATS 1654 set _C_LABEL(wfill), %g1 1655 lduw [%g1], %g5 1656 inc %g5 1657 stw %g5, [%g1] 1658#endif 1659 btst TSTATE_PRIV, %g4 ! User mode? 1660 and %g4, CWP, %g5 ! %g4 = %cwp of trap 1661 wrpr %g7, 0, %tt 1662 bz,a,pt %icc, datafault ! We were in user mode -- normal fault 1663 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consisent 1664 1665 /* 1666 * We're in a pickle here. We were trying to return to user mode 1667 * and the restore of the user window failed, so now we have one valid 1668 * kernel window and a user window state. If we do a TRAP_SETUP() now, 1669 * our kernel window will be considered a user window and cause a 1670 * fault when we try to save it later due to an invalid user address. 1671 * If we return to where we faulted, our window state will not be valid 1672 * and we will fault trying to enter user with our primary context of zero. 1673 * 1674 * What we'll do is arrange to have us return to return_from_trap so we will 1675 * start the whole business over again. But first, switch to a kernel window 1676 * setup. Let's see, canrestore and otherwin are zero. Set WSTATE_KERN and 1677 * make sure we're in kernel context and we're done. 1678 */ 1679 1680#ifdef TRAPSTATS 1681 set _C_LABEL(kwfill), %g4 1682 lduw [%g4], %g7 1683 inc %g7 1684 stw %g7, [%g4] 1685#endif 1686#if 0 /* Need to switch over to new stuff to fix WDR bug */ 1687 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consisent 1688 wrpr %g2, %g0, %tl ! Restore trap level -- we need to reuse it 1689 set return_from_trap, %g4 1690 set CTX_PRIMARY, %g7 1691 wrpr %g4, 0, %tpc 1692 stxa %g0, [%g7] ASI_DMMU 1693 inc 4, %g4 1694 membar #Sync 1695 flush %g4 ! Isn't this convenient? 1696 wrpr %g0, WSTATE_KERN, %wstate 1697 wrpr %g0, 0, %canrestore ! These should be zero but 1698 wrpr %g0, 0, %otherwin ! clear them just in case 1699 rdpr %ver, %g5 1700 and %g5, CWP, %g5 1701 wrpr %g0, 0, %cleanwin 1702 dec 1, %g5 ! NWINDOWS-1-1 1703 wrpr %g5, 0, %cansave ! Invalidate all windows 1704! flushw ! DEBUG 1705 ba,pt %icc, datafault 1706 wrpr %g4, 0, %tnpc 1707#else 1708 wrpr %g2, %g0, %tl ! Restore trap level 1709 cmp %g2, 3 1710 tne %icc, 1 1711 rdpr %tt, %g5 1712 wrpr %g0, 1, %tl ! Revert to TL==1 XXX what if this wasn't in rft_user? Oh well. 1713 wrpr %g5, %g0, %tt ! Set trap type correctly 1714/* 1715 * Here we need to implement the beginning of datafault. 1716 * TRAP_SETUP expects to come from either kernel mode or 1717 * user mode with at least one valid register window. It 1718 * will allocate a trap frame, save the out registers, and 1719 * fix the window registers to think we have one user 1720 * register window. 1721 * 1722 * However, under these circumstances we don't have any 1723 * valid register windows, so we need to clean up the window 1724 * registers to prevent garbage from being saved to either 1725 * the user stack or the PCB before calling the datafault 1726 * handler. 1727 * 1728 * We could simply jump to datafault if we could somehow 1729 * make the handler issue a `saved' instruction immediately 1730 * after creating the trapframe. 1731 * 1732 * The following is duplicated from datafault: 1733 */ 1734 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to AG regs 1735#ifdef TRAPS_USE_IG 1736 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to AG regs 1737#endif 1738 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 1739 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 1740 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 1741 ldxa [SFSR] %asi, %g3 ! get sync fault status register 1742 stxa %g0, [SFSR] %asi ! Clear out fault now 1743 1744 TRAP_SETUP(-CC64FSZ-TF_SIZE) 1745 saved ! Blow away that one register window we didn't ever use. 1746 ba,a,pt %icc, Ldatafault_internal ! Now we should return directly to user mode 1747 nop 1748#endif 1749winfixspill: 1750 bne,a,pt %xcc, datafault ! Was not a spill -- handle it normally 1751 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 1752 1753 !! 1754 !! This was a spill 1755 !! 1756#if 1 1757 btst TSTATE_PRIV, %g4 ! From user mode? 1758 wrpr %g2, 0, %tl ! We need to load the fault type so we can 1759 rdpr %tt, %g5 ! overwrite the lower trap and get it to the fault handler 1760 wrpr %g1, 0, %tl 1761 wrpr %g5, 0, %tt ! Copy over trap type for the fault handler 1762 and %g4, CWP, %g5 ! find %cwp from trap 1763 be,a,pt %xcc, datafault ! Let's do a regular datafault. When we try a save in datafault we'll 1764 wrpr %g5, 0, %cwp ! return here and write out all dirty windows. 1765#endif 1766 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 1767 LDPTR [%g6 + %lo(CPCB)], %g6 ! This is in the locked TLB and should not fault 1768#ifdef TRAPSTATS 1769 set _C_LABEL(wspill), %g7 1770 lduw [%g7], %g5 1771 inc %g5 1772 stw %g5, [%g7] 1773#endif 1774 1775 /* 1776 * Traverse kernel map to find paddr of cpcb and only us ASI_PHYS_CACHED to 1777 * prevent any faults while saving the windows. BTW if it isn't mapped, we 1778 * will trap and hopefully panic. 1779 */ 1780 1781! ba 0f ! DEBUG -- don't use phys addresses 1782 wr %g0, ASI_NUCLEUS, %asi ! In case of problems finding PA 1783 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g1 1784 LDPTR [%g1 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g1 ! Load start of ctxbusy 1785#ifdef DEBUG 1786 srax %g6, HOLESHIFT, %g7 ! Check for valid address 1787 brz,pt %g7, 1f ! Should be zero or -1 1788 addcc %g7, 1, %g7 ! Make -1 -> 0 1789 tnz %xcc, 1 ! Invalid address??? How did this happen? 17901: 1791#endif 1792 srlx %g6, STSHIFT, %g7 1793 ldx [%g1], %g1 ! Load pointer to kernel_pmap 1794 and %g7, STMASK, %g7 1795 sll %g7, 3, %g7 1796 add %g7, %g1, %g1 1797 DLFLUSH(%g1,%g7) 1798 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 1799 DLFLUSH2(%g7) 1800 1801 srlx %g6, PDSHIFT, %g7 ! Do page directory 1802 and %g7, PDMASK, %g7 1803 sll %g7, 3, %g7 1804 brz,pn %g1, 0f 1805 add %g7, %g1, %g1 1806 DLFLUSH(%g1,%g7) 1807 ldxa [%g1] ASI_PHYS_CACHED, %g1 1808 DLFLUSH2(%g7) 1809 1810 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 1811 and %g7, PTMASK, %g7 1812 brz %g1, 0f 1813 sll %g7, 3, %g7 1814 add %g1, %g7, %g7 1815 DLFLUSH(%g7,%g1) 1816 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 1817 DLFLUSH2(%g1) 1818 brgez %g7, 0f 1819 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 1820 sll %g6, 32-PGSHIFT, %g6 ! And offset 1821 sllx %g7, PGSHIFT+23, %g7 ! There are 23 bits to the left of the PA in the TTE 1822 srl %g6, 32-PGSHIFT, %g6 1823 srax %g7, 23, %g7 1824 or %g7, %g6, %g6 ! Then combine them to form PA 1825 1826 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 18270: 1828 /* 1829 * Now save all user windows to cpcb. 1830 */ 1831#ifdef NOTDEF_DEBUG 1832 add %g6, PCB_NSAVED, %g7 1833 DLFLUSH(%g7,%g5) 1834 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! make sure that pcb_nsaved 1835 DLFLUSH2(%g5) 1836 brz,pt %g7, 1f ! is zero, else 1837 nop 1838 wrpr %g0, 4, %tl 1839 sir ! Force a watchdog 18401: 1841#endif 1842 rdpr %otherwin, %g7 1843 brnz,pt %g7, 1f 1844 rdpr %canrestore, %g5 1845 rdpr %cansave, %g1 1846 add %g5, 1, %g7 ! add the %cwp window to the list to save 1847! movrnz %g1, %g5, %g7 ! If we're issuing a save 1848! mov %g5, %g7 ! DEBUG 1849 wrpr %g0, 0, %canrestore 1850 wrpr %g7, 0, %otherwin ! Still in user mode -- need to switch to kernel mode 18511: 1852 mov %g7, %g1 1853 add %g6, PCB_NSAVED, %g7 1854 DLFLUSH(%g7,%g5) 1855 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Start incrementing pcb_nsaved 1856 DLFLUSH2(%g5) 1857 1858#ifdef DEBUG 1859 wrpr %g0, 5, %tl 1860#endif 1861 mov %g6, %g5 1862 brz,pt %g7, winfixsave ! If it's in use, panic 1863 saved ! frob window registers 1864 1865 /* PANIC */ 1866! sir ! Force a watchdog 1867#ifdef DEBUG 1868 wrpr %g2, 0, %tl 1869#endif 1870 mov %g7, %o2 1871 rdpr %ver, %o1 1872 sethi %hi(2f), %o0 1873 and %o1, CWP, %o1 1874 wrpr %g0, %o1, %cleanwin 1875 dec 1, %o1 1876 wrpr %g0, %o1, %cansave ! kludge away any more window problems 1877 wrpr %g0, 0, %canrestore 1878 wrpr %g0, 0, %otherwin 1879 or %lo(2f), %o0, %o0 1880 wrpr %g0, WSTATE_KERN, %wstate 1881 sethi %hi(PANICSTACK), %sp 1882 LDPTR [%sp + %lo(PANICSTACK)], %sp 1883 add %sp, -CC64FSZ-STKB, %sp 1884 ta 1; nop ! This helps out traptrace. 1885 call _C_LABEL(panic) ! This needs to be fixed properly but we should panic here 1886 mov %g1, %o1 1887 NOTREACHED 1888 .data 18892: 1890 .asciz "winfault: double invalid window at %p, nsaved=%d" 1891 _ALIGN 1892 .text 18933: 1894 saved 1895 save 1896winfixsave: 1897 stxa %l0, [%g5 + PCB_RW + ( 0*8)] %asi ! Save the window in the pcb, we can schedule other stuff in here 1898 stxa %l1, [%g5 + PCB_RW + ( 1*8)] %asi 1899 stxa %l2, [%g5 + PCB_RW + ( 2*8)] %asi 1900 stxa %l3, [%g5 + PCB_RW + ( 3*8)] %asi 1901 stxa %l4, [%g5 + PCB_RW + ( 4*8)] %asi 1902 stxa %l5, [%g5 + PCB_RW + ( 5*8)] %asi 1903 stxa %l6, [%g5 + PCB_RW + ( 6*8)] %asi 1904 stxa %l7, [%g5 + PCB_RW + ( 7*8)] %asi 1905 1906 stxa %i0, [%g5 + PCB_RW + ( 8*8)] %asi 1907 stxa %i1, [%g5 + PCB_RW + ( 9*8)] %asi 1908 stxa %i2, [%g5 + PCB_RW + (10*8)] %asi 1909 stxa %i3, [%g5 + PCB_RW + (11*8)] %asi 1910 stxa %i4, [%g5 + PCB_RW + (12*8)] %asi 1911 stxa %i5, [%g5 + PCB_RW + (13*8)] %asi 1912 stxa %i6, [%g5 + PCB_RW + (14*8)] %asi 1913 stxa %i7, [%g5 + PCB_RW + (15*8)] %asi 1914 1915! rdpr %otherwin, %g1 ! Check to see if we's done 1916 dec %g1 1917 wrpr %g0, 7, %cleanwin ! BUGBUG -- we should not hardcode this, but I have no spare globals 1918 inc 16*8, %g5 ! Move to next window 1919 inc %g7 ! inc pcb_nsaved 1920 brnz,pt %g1, 3b 1921 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 1922 1923 /* fix up pcb fields */ 1924 stba %g7, [%g6 + PCB_NSAVED] %asi ! cpcb->pcb_nsaved = n 1925#if 0 1926 mov %g7, %g5 ! fixup window registers 19275: 1928 dec %g5 1929 brgz,a,pt %g5, 5b 1930 restore 1931#ifdef NOT_DEBUG 1932 rdpr %wstate, %g5 ! DEBUG 1933 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 1934 wrpr %g0, 4, %tl 1935 rdpr %cansave, %g7 1936 rdpr %canrestore, %g6 1937 flushw ! DEBUG 1938 wrpr %g2, 0, %tl 1939 wrpr %g5, 0, %wstate ! DEBUG 1940#endif 1941#else 1942 /* 1943 * We just issued a bunch of saves, so %cansave is now 0, 1944 * probably (if we were doing a flushw then we may have 1945 * come in with only partially full register windows and 1946 * it may not be 0). 1947 * 1948 * %g7 contains the count of the windows we just finished 1949 * saving. 1950 * 1951 * What we need to do now is move some of the windows from 1952 * %canrestore to %cansave. What we should do is take 1953 * min(%canrestore, %g7) and move that over to %cansave. 1954 * 1955 * %g7 is the number of windows we flushed, so we should 1956 * use that as a base. Clear out %otherwin, set %cansave 1957 * to min(%g7, NWINDOWS - 2), set %cleanwin to %canrestore 1958 * + %cansave and the rest follows: 1959 * 1960 * %otherwin = 0 1961 * %cansave = NWINDOWS - 2 - %canrestore 1962 */ 1963 wrpr %g0, 0, %otherwin 1964 rdpr %canrestore, %g1 1965 sub %g1, %g7, %g1 ! Calculate %canrestore - %g7 1966 movrlz %g1, %g0, %g1 ! Clamp at zero 1967 wrpr %g1, 0, %canrestore ! This is the new canrestore 1968 rdpr %ver, %g5 1969 and %g5, CWP, %g5 ! NWINDOWS-1 1970 dec %g5 ! NWINDOWS-2 1971 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 1972 sub %g5, %g1, %g5 ! NWINDOWS-2-%canrestore 1973 wrpr %g5, 0, %cansave 1974#ifdef NOT_DEBUG 1975 rdpr %wstate, %g5 ! DEBUG 1976 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 1977 wrpr %g0, 4, %tl 1978 flushw ! DEBUG 1979 wrpr %g2, 0, %tl 1980 wrpr %g5, 0, %wstate ! DEBUG 1981#endif 1982#endif 1983 1984#ifdef NOTDEF_DEBUG 1985 set panicstack-CC64FSZ, %g1 1986 save %g1, 0, %sp 1987 GLOBTOLOC 1988 rdpr %wstate, %l0 1989 wrpr %g0, WSTATE_KERN, %wstate 1990 set 8f, %o0 1991 mov %g7, %o1 1992 call printf 1993 mov %g5, %o2 1994 wrpr %l0, 0, %wstate 1995 LOCTOGLOB 1996 restore 1997 .data 19988: 1999 .asciz "winfix: spill fixup\n" 2000 _ALIGN 2001 .text 2002#endif 2003! rdpr %tl, %g2 ! DEBUG DEBUG -- did we trap somewhere? 2004 sub %g2, 1, %g1 2005 rdpr %tt, %g2 2006 wrpr %g1, 0, %tl ! We will not attempt to re-execute the spill, so dump our trap frame permanently 2007 wrpr %g2, 0, %tt ! Move trap type from fault frame here, overwriting spill 2008 2009 /* Did we save a user or kernel window ? */ 2010! srax %g3, 48, %g5 ! User or kernel store? (TAG TARGET) 2011 sllx %g3, (64-13), %g5 ! User or kernel store? (TAG ACCESS) 2012 sethi %hi(dcache_size), %g7 2013 ld [%g7 + %lo(dcache_size)], %g7 2014 sethi %hi(dcache_line_size), %g6 2015 ld [%g6 + %lo(dcache_line_size)], %g6 2016 brnz,pt %g5, 1f ! User fault -- save windows to pcb 2017 sub %g7, %g6, %g7 2018 2019 and %g4, CWP, %g4 ! %g4 = %cwp of trap 2020 wrpr %g4, 0, %cwp ! Kernel fault -- restore %cwp and force and trap to debugger 2021 !! 2022 !! Here we managed to fault trying to access a kernel window 2023 !! This is a bug. Switch to the interrupt stack if we aren't 2024 !! there already and then trap into the debugger or panic. 2025 !! 2026 sethi %hi(EINTSTACK-BIAS), %g6 2027 btst 1, %sp 2028 bnz,pt %icc, 0f 2029 mov %sp, %g1 2030 add %sp, -BIAS, %g1 20310: 2032 or %g6, %lo(EINTSTACK-BIAS), %g6 2033 set (EINTSTACK-INTSTACK), %g7 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses 2034 sub %g6, %g1, %g2 ! Determine if we need to switch to intr stack or not 2035 dec %g7 ! Make it into a mask 2036 andncc %g2, %g7, %g0 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 2037 movz %xcc, %g1, %g6 ! Stay on interrupt stack? 2038 add %g6, -CCFSZ, %g6 ! Allocate a stack frame 2039 mov %sp, %l6 ! XXXXX Save old stack pointer 2040 mov %g6, %sp 2041 ta 1; nop ! Enter debugger 2042 NOTREACHED 20431: 2044#if 1 2045 /* Now we need to blast away the D$ to make sure we're in sync */ 2046 stxa %g0, [%g7] ASI_DCACHE_TAG 2047 brnz,pt %g7, 1b 2048 sub %g7, %g6, %g7 2049#endif 2050 2051#ifdef NOTDEF_DEBUG 2052 set panicstack-CC64FSZ, %g5 2053 save %g5, 0, %sp 2054 GLOBTOLOC 2055 rdpr %wstate, %l0 2056 wrpr %g0, WSTATE_KERN, %wstate 2057 set 8f, %o0 2058 call printf 2059 mov %fp, %o1 2060 wrpr %l0, 0, %wstate 2061 LOCTOGLOB 2062 restore 2063 .data 20648: 2065 .asciz "winfix: kernel spill retry\n" 2066 _ALIGN 2067 .text 2068#endif 2069#ifdef TRAPSTATS 2070 set _C_LABEL(wspillskip), %g4 2071 lduw [%g4], %g5 2072 inc %g5 2073 stw %g5, [%g4] 2074#endif 2075 /* 2076 * If we had WSTATE_KERN then we had at least one valid kernel window. 2077 * We should re-execute the trapping save. 2078 */ 2079 rdpr %wstate, %g3 2080 mov %g3, %g3 2081 cmp %g3, WSTATE_KERN 2082 bne,pt %icc, 1f 2083 nop 2084 retry ! Now we can complete the save 20851: 2086 /* 2087 * Since we had a WSTATE_USER, we had no valid kernel windows. This should 2088 * only happen inside TRAP_SETUP or INTR_SETUP. Emulate 2089 * the instruction, clean up the register windows, then done. 2090 */ 2091 rdpr %cwp, %g1 2092 inc %g1 2093 rdpr %tstate, %g2 2094 wrpr %g1, %cwp 2095 andn %g2, CWP, %g2 2096 wrpr %g1, %g2, %tstate 2097 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate 2098#ifdef TRAPS_USE_IG 2099 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 2100#endif 2101 mov %g6, %sp 2102 done 2103 2104/* 2105 * Each memory data access fault, from user or kernel mode, 2106 * comes here. 2107 * 2108 * We will assume that %pil is not lost so we won't bother to save it 2109 * unless we're in an interrupt handler. 2110 * 2111 * On entry: 2112 * We are on one of the alternate set of globals 2113 * %g1 = MMU tag target 2114 * %g2 = %tl 2115 * 2116 * On return: 2117 * 2118 */ 2119datafault: 2120 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to AG regs 2121#ifdef TRAPS_USE_IG 2122 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to AG regs 2123#endif 2124 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 2125 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2126 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 2127 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2128 stxa %g0, [SFSR] %asi ! Clear out fault now 2129 2130 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2131Ldatafault_internal: 2132 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2133! ldx [%sp + CC64FSZ + STKB + TF_FAULT], %g1 ! DEBUG make sure this has not changed 2134 mov %g1, %o0 ! Move these to the out regs so we can save the globals 2135 mov %g2, %o4 2136 mov %g3, %o5 2137 2138 ldxa [%g0] ASI_AFAR, %o2 ! get async fault address 2139 ldxa [%g0] ASI_AFSR, %o3 ! get async fault status 2140 mov -1, %g7 2141 stxa %g7, [%g0] ASI_AFSR ! And clear this out, too 2142 2143 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 2144 2145 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2146 rdpr %tt, %o1 ! find out what trap brought us here 2147 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2148 rdpr %tstate, %g1 2149 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2150 rdpr %tpc, %g2 2151 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2152 rdpr %tnpc, %g3 2153 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2154 mov %g2, %o7 ! Make the fault address look like the return address 2155 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2156 rd %y, %g5 ! save y 2157 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2158 2159 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] 2160 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2161 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] ! set tf.tf_npc 2162 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 2163 2164 rdpr %pil, %g4 2165 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2166 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2167 2168#if 1 2169 rdpr %tl, %g7 2170 dec %g7 2171 movrlz %g7, %g0, %g7 2172 wrpr %g0, %g7, %tl ! Revert to kernel mode 2173#else 2174 wrpr %g0, 0, %tl ! Revert to kernel mode 2175#endif 2176 /* Finish stackframe, call C trap handler */ 2177 flushw ! Get this clean so we won't take any more user faults 2178#ifdef NOTDEF_DEBUG 2179 set CPCB, %o7 2180 LDPTR [%o7], %o7 2181 ldub [%o7 + PCB_NSAVED], %o7 2182 brz,pt %o7, 2f 2183 nop 2184 save %sp, -CC64FSZ, %sp 2185 set 1f, %o0 2186 call printf 2187 mov %i7, %o1 2188 ta 1; nop 2189 restore 2190 .data 21911: .asciz "datafault: nsaved = %d\n" 2192 _ALIGN 2193 .text 21942: 2195#endif 2196 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2197 !! In our case we need to clear it before calling any C-code 2198 clr %g4 2199 2200 /* 2201 * Right now the registers have the following values: 2202 * 2203 * %o0 -- MMU_TAG_ACCESS 2204 * %o1 -- TT 2205 * %o2 -- afar 2206 * %o3 -- afsr 2207 * %o4 -- sfar 2208 * %o5 -- sfsr 2209 */ 2210 2211 cmp %o1, T_DATA_ERROR 2212 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 2213 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2214 be,pn %icc, data_error 2215 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2216 2217 mov %o0, %o3 ! (argument: trap address) 2218 mov %g2, %o2 ! (argument: trap pc) 2219 call _C_LABEL(data_access_fault) ! data_access_fault(&tf, type, 2220 ! pc, addr, sfva, sfsr) 2221 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2222 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2223 2224data_recover: 2225#ifdef TRAPSTATS 2226 set _C_LABEL(uintrcnt), %g1 2227 stw %g0, [%g1] 2228 set _C_LABEL(iveccnt), %g1 2229 stw %g0, [%g1] 2230#endif 2231 b return_from_trap ! go return 2232 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2233 NOTREACHED 2234 2235data_error: 2236 call _C_LABEL(data_access_error) ! data_access_error(&tf, type, 2237 ! afva, afsr, sfva, sfsr) 2238 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2239 ba data_recover 2240 nop 2241 NOTREACHED 2242 2243/* 2244 * Each memory instruction access fault from a fast access handler comes here. 2245 * We will quickly check if this is an original prom mapping before going 2246 * to the generic fault handler 2247 * 2248 * We will assume that %pil is not lost so we won't bother to save it 2249 * unless we're in an interrupt handler. 2250 * 2251 * On entry: 2252 * We are on one of the alternate set of globals 2253 * %g1 = MMU tag target 2254 * %g2 = TSB entry ptr 2255 * %g3 = TLB Tag Access 2256 * 2257 * On return: 2258 * 2259 */ 2260 2261 ICACHE_ALIGN 2262instr_miss: 2263#ifdef TRAPSTATS 2264 set _C_LABEL(ktmiss), %g3 2265 set _C_LABEL(utmiss), %g4 2266 rdpr %tl, %g6 2267 dec %g6 2268 movrz %g6, %g4, %g3 2269 lduw [%g3], %g4 2270 inc %g4 2271 stw %g4, [%g3] 2272#endif 2273 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 2274 sethi %hi(0x1fff), %g7 ! 8K context mask 2275 ldxa [%g3] ASI_IMMU, %g3 ! from tag access register 2276 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 2277 or %g7, %lo(0x1fff), %g7 2278 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 2279 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2280 and %g3, %g7, %g6 ! Isolate context 2281 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2282 inc %g5 ! (0 or -1) -> (1 or 0) 2283 ldx [%g4+%g6], %g4 ! Load up our page table. 2284#ifdef DEBUG 2285 /* Make sure we don't try to replace a kernel translation */ 2286 /* This should not be necessary */ 2287 brnz,pt %g6, 1f ! If user context continue miss 2288 sethi %hi(KERNBASE), %g7 ! Don't need %lo 2289 set 0x0800000, %g6 ! 8MB 2290 sub %g3, %g7, %g7 2291 cmp %g7, %g6 2292 tlu %xcc, 1; nop 22931: 2294#endif 2295 srlx %g3, STSHIFT, %g6 2296 cmp %g5, 1 2297 bgu,pn %xcc, textfault ! Error! 2298 srlx %g3, PDSHIFT, %g5 2299 and %g6, STMASK, %g6 2300 sll %g6, 3, %g6 2301 and %g5, PDMASK, %g5 2302 nop 2303 2304 sll %g5, 3, %g5 2305 add %g6, %g4, %g4 2306 ldxa [%g4] ASI_PHYS_CACHED, %g4 2307 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2308 and %g6, PTMASK, %g6 2309 add %g5, %g4, %g5 2310 brz,pn %g4, textfault ! NULL entry? check somewhere else 2311 nop 2312 2313 ldxa [%g5] ASI_PHYS_CACHED, %g4 2314 sll %g6, 3, %g6 2315 brz,pn %g4, textfault ! NULL entry? check somewhere else 2316 add %g6, %g4, %g6 23171: 2318 ldxa [%g6] ASI_PHYS_CACHED, %g4 2319 brgez,pn %g4, textfault 2320 nop 2321 2322 /* Check if it's an executable mapping. */ 2323 andcc %g4, TTE_EXEC, %g0 2324 bz,pn %xcc, textfault 2325 nop 2326 2327 or %g4, TTE_ACCESS, %g7 ! Update accessed bit 2328 btst TTE_ACCESS, %g4 ! Need to update access git? 2329 bne,pt %xcc, 1f 2330 nop 2331 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and store it 2332 cmp %g4, %g7 2333 bne,pn %xcc, 1b 2334 or %g4, TTE_ACCESS, %g4 ! Update accessed bit 23351: 2336 stx %g1, [%g2] ! Update TSB entry tag 2337 stx %g4, [%g2+8] ! Update TSB entry data 2338 stxa %g4, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 2339 membar #Sync 2340 CLRTT 2341 retry 2342 NOTREACHED 2343 !! 2344 !! Check our prom mappings -- temporary 2345 !! 2346 2347/* 2348 * Each memory text access fault, from user or kernel mode, 2349 * comes here. 2350 * 2351 * We will assume that %pil is not lost so we won't bother to save it 2352 * unless we're in an interrupt handler. 2353 * 2354 * On entry: 2355 * We are on one of the alternate set of globals 2356 * %g1 = MMU tag target 2357 * %g2 = %tl 2358 * %g3 = %tl - 1 2359 * 2360 * On return: 2361 * 2362 */ 2363 2364textfault: 2365 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to AG regs 2366#ifdef TRAPS_USE_IG 2367 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to AG regs 2368#endif 2369 wr %g0, ASI_IMMU, %asi 2370 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2371 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2372 membar #LoadStore 2373 stxa %g0, [SFSR] %asi ! Clear out old info 2374 2375 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2376 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2377 2378 mov %g3, %o3 2379 2380 wrpr %g0, PSTATE_KERN, %pstate ! Switch to normal globals 2381 ldxa [%g0] ASI_AFSR, %o4 ! get async fault status 2382 ldxa [%g0] ASI_AFAR, %o5 ! get async fault address 2383 mov -1, %o0 2384 stxa %o0, [%g0] ASI_AFSR ! Clear this out 2385 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2386 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2387 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2388 rdpr %tt, %o1 ! Find out what caused this trap 2389 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2390 rdpr %tstate, %g1 2391 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2392 rdpr %tpc, %o2 ! sync virt addr; must be read first 2393 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2394 rdpr %tnpc, %g3 2395 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2396 rd %y, %g5 ! save y 2397 2398 /* Finish stackframe, call C trap handler */ 2399 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2400 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] ! debug 2401 2402 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 2403 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] ! set tf.tf_npc 2404 2405 rdpr %pil, %g4 2406 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2407 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2408 2409 rdpr %tl, %g7 2410 dec %g7 2411 movrlz %g7, %g0, %g7 2412 wrpr %g0, %g7, %tl ! Revert to kernel mode 2413 2414 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2415 flushw ! Get rid of any user windows so we don't deadlock 2416 2417 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2418 !! In our case we need to clear it before calling any C-code 2419 clr %g4 2420 2421 /* Use trap type to see what handler to call */ 2422 cmp %o1, T_INST_ERROR 2423 be,pn %xcc, text_error 2424 st %g5, [%sp + CC64FSZ + STKB + TF_Y] ! set tf.tf_y 2425 2426 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2427 call _C_LABEL(text_access_fault) ! mem_access_fault(&tf, type, pc, sfsr) 2428 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2429text_recover: 2430 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2431 b return_from_trap ! go return 2432 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2433 NOTREACHED 2434 2435text_error: 2436 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2437 call _C_LABEL(text_access_error) ! mem_access_fault(&tfm type, sfva [pc], sfsr, 2438 ! afva, afsr); 2439 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2440 ba text_recover 2441 nop 2442 NOTREACHED 2443 2444/* 2445 * We're here because we took an alignment fault in NUCLEUS context. 2446 * This could be a kernel bug or it could be due to saving a user 2447 * window to an invalid stack pointer. 2448 * 2449 * If the latter is the case, we could try to emulate unaligned accesses, 2450 * but we really don't know where to store the registers since we can't 2451 * determine if there's a stack bias. Or we could store all the regs 2452 * into the PCB and punt, until the user program uses up all the CPU's 2453 * register windows and we run out of places to store them. So for 2454 * simplicity we'll just blow them away and enter the trap code which 2455 * will generate a bus error. Debugging the problem will be a bit 2456 * complicated since lots of register windows will be lost, but what 2457 * can we do? 2458 */ 2459checkalign: 2460 rdpr %tl, %g2 2461 subcc %g2, 1, %g1 2462 bneg,pn %icc, slowtrap ! Huh? 2463 sethi %hi(CPCB), %g6 ! get current pcb 2464 2465 wrpr %g1, 0, %tl 2466 rdpr %tt, %g7 2467 rdpr %tstate, %g4 2468 andn %g7, 0x3f, %g5 2469 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 2470 bne,a,pn %icc, slowtrap 2471 wrpr %g1, 0, %tl ! Revert TL XXX wrpr in a delay slot... 2472 2473#ifdef DEBUG 2474 cmp %g7, 0x34 ! If we took a datafault just before this trap 2475 bne,pt %icc, checkalignspill ! our stack's probably bad so we need to switch somewhere else 2476 nop 2477 2478 !! 2479 !! Double data fault -- bad stack? 2480 !! 2481 wrpr %g2, %tl ! Restore trap level. 2482 sir ! Just issue a reset and don't try to recover. 2483 mov %fp, %l6 ! Save the frame pointer 2484 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 2485 add %fp, -CC64FSZ, %sp ! Create a stackframe 2486 wrpr %g0, 15, %pil ! Disable interrupts, too 2487 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 2488 wrpr %g0, 7, %cansave ! probably is too, so blow away 2489 ba slowtrap ! all our register windows. 2490 wrpr %g0, 0x101, %tt 2491#endif 2492checkalignspill: 2493 /* 2494 * %g1 -- current tl 2495 * %g2 -- original tl 2496 * %g4 -- tstate 2497 * %g7 -- tt 2498 */ 2499 2500 and %g4, CWP, %g5 2501 wrpr %g5, %cwp ! Go back to the original register win 2502 2503 /* 2504 * Remember: 2505 * 2506 * %otherwin = 0 2507 * %cansave = NWINDOWS - 2 - %canrestore 2508 */ 2509 2510 rdpr %otherwin, %g6 2511 rdpr %canrestore, %g3 2512 rdpr %ver, %g5 2513 sub %g3, %g6, %g3 ! Calculate %canrestore - %g7 2514 and %g5, CWP, %g5 ! NWINDOWS-1 2515 movrlz %g3, %g0, %g3 ! Clamp at zero 2516 wrpr %g0, 0, %otherwin 2517 wrpr %g3, 0, %canrestore ! This is the new canrestore 2518 dec %g5 ! NWINDOWS-2 2519 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 2520 sub %g5, %g3, %g5 ! NWINDOWS-2-%canrestore 2521 wrpr %g5, 0, %cansave 2522 2523 wrpr %g0, T_ALIGN, %tt ! This was an alignment fault 2524 /* 2525 * Now we need to determine if this was a userland store or not. 2526 * Userland stores occur in anything other than the kernel spill 2527 * handlers (trap type 09x). 2528 */ 2529 and %g7, 0xff0, %g5 2530 cmp %g5, 0x90 2531 bz,pn %icc, slowtrap 2532 nop 2533 bclr TSTATE_PRIV, %g4 2534 wrpr %g4, 0, %tstate 2535 ba,a,pt %icc, slowtrap 2536 nop 2537 2538/* 2539 * slowtrap() builds a trap frame and calls trap(). 2540 * This is called `slowtrap' because it *is*.... 2541 * We have to build a full frame for ptrace(), for instance. 2542 * 2543 * Registers: 2544 * 2545 */ 2546slowtrap: 2547#ifdef TRAPS_USE_IG 2548 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 2549#endif 2550#ifdef DIAGNOSTIC 2551 /* Make sure kernel stack is aligned */ 2552 btst 0x03, %sp ! 32-bit stack OK? 2553 and %sp, 0x07, %g4 ! 64-bit stack OK? 2554 bz,pt %icc, 1f 2555 cmp %g4, 0x1 ! Must end in 0b001 2556 be,pt %icc, 1f 2557 rdpr %wstate, %g7 2558 cmp %g7, WSTATE_KERN 2559 bnz,pt %icc, 1f ! User stack -- we'll blow it away 2560 nop 2561 sethi %hi(PANICSTACK), %sp 2562 LDPTR [%sp + %lo(PANICSTACK)], %sp 2563 add %sp, -CC64FSZ-STKB, %sp 25641: 2565#endif 2566 rdpr %tt, %g4 2567 rdpr %tstate, %g1 2568 rdpr %tpc, %g2 2569 rdpr %tnpc, %g3 2570 2571 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2572Lslowtrap_reenter: 2573 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 2574 mov %g4, %o1 ! (type) 2575 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] 2576 rd %y, %g5 2577 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 2578 mov %g1, %o3 ! (pstate) 2579 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 2580 mov %g2, %o2 ! (pc) 2581 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 2582 2583 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 2584 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] 2585 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] 2586 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 2587 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] 2588 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] 2589 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] 2590 rdpr %pil, %g5 2591 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] 2592 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] 2593 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 2594 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2595 /* 2596 * Phew, ready to enable traps and call C code. 2597 */ 2598 rdpr %tl, %g1 2599 dec %g1 2600 movrlz %g1, %g0, %g1 2601 wrpr %g0, %g1, %tl ! Revert to kernel mode 2602 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2603 !! In our case we need to clear it before calling any C-code 2604 clr %g4 2605 2606 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2607 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 2608 call _C_LABEL(trap) ! trap(tf, type, pc, pstate) 2609 nop 2610 2611 ba,a,pt %icc, return_from_trap 2612 nop 2613 NOTREACHED 2614#if 1 2615/* 2616 * This code is no longer needed. 2617 */ 2618/* 2619 * Do a `software' trap by re-entering the trap code, possibly first 2620 * switching from interrupt stack to kernel stack. This is used for 2621 * scheduling and signal ASTs (which generally occur from softclock or 2622 * tty or net interrupts). 2623 * 2624 * We enter with the trap type in %g1. All we have to do is jump to 2625 * Lslowtrap_reenter above, but maybe after switching stacks.... 2626 * 2627 * We should be running alternate globals. The normal globals and 2628 * out registers were just loaded from the old trap frame. 2629 * 2630 * Input Params: 2631 * %g1 = tstate 2632 * %g2 = tpc 2633 * %g3 = tnpc 2634 * %g4 = tt == T_AST 2635 */ 2636softtrap: 2637 sethi %hi(EINTSTACK-STKB), %g5 2638 sethi %hi(EINTSTACK-INTSTACK), %g7 2639 or %g5, %lo(EINTSTACK-STKB), %g5 2640 dec %g7 2641 sub %g5, %sp, %g5 2642 sethi %hi(CPCB), %g6 2643 andncc %g5, %g7, %g0 2644 bnz,pt %xcc, Lslowtrap_reenter 2645 LDPTR [%g6 + %lo(CPCB)], %g7 2646 set USPACE-CC64FSZ-TF_SIZE-STKB, %g5 2647 add %g7, %g5, %g6 2648 SET_SP_REDZONE(%g7, %g5) 2649#ifdef DEBUG 2650 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT] ! Generate a new trapframe 2651#endif 2652 stx %i0, [%g6 + CC64FSZ + STKB + TF_O + (0*8)] ! but don't bother with 2653 stx %i1, [%g6 + CC64FSZ + STKB + TF_O + (1*8)] ! locals and ins 2654 stx %i2, [%g6 + CC64FSZ + STKB + TF_O + (2*8)] 2655 stx %i3, [%g6 + CC64FSZ + STKB + TF_O + (3*8)] 2656 stx %i4, [%g6 + CC64FSZ + STKB + TF_O + (4*8)] 2657 stx %i5, [%g6 + CC64FSZ + STKB + TF_O + (5*8)] 2658 stx %i6, [%g6 + CC64FSZ + STKB + TF_O + (6*8)] 2659 stx %i7, [%g6 + CC64FSZ + STKB + TF_O + (7*8)] 2660#ifdef DEBUG 2661 ldx [%sp + CC64FSZ + STKB + TF_I + (0*8)], %l0 ! Copy over the rest of the regs 2662 ldx [%sp + CC64FSZ + STKB + TF_I + (1*8)], %l1 ! But just dirty the locals 2663 ldx [%sp + CC64FSZ + STKB + TF_I + (2*8)], %l2 2664 ldx [%sp + CC64FSZ + STKB + TF_I + (3*8)], %l3 2665 ldx [%sp + CC64FSZ + STKB + TF_I + (4*8)], %l4 2666 ldx [%sp + CC64FSZ + STKB + TF_I + (5*8)], %l5 2667 ldx [%sp + CC64FSZ + STKB + TF_I + (6*8)], %l6 2668 ldx [%sp + CC64FSZ + STKB + TF_I + (7*8)], %l7 2669 stx %l0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)] 2670 stx %l1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)] 2671 stx %l2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)] 2672 stx %l3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)] 2673 stx %l4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)] 2674 stx %l5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)] 2675 stx %l6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)] 2676 stx %l7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)] 2677 ldx [%sp + CC64FSZ + STKB + TF_L + (0*8)], %l0 2678 ldx [%sp + CC64FSZ + STKB + TF_L + (1*8)], %l1 2679 ldx [%sp + CC64FSZ + STKB + TF_L + (2*8)], %l2 2680 ldx [%sp + CC64FSZ + STKB + TF_L + (3*8)], %l3 2681 ldx [%sp + CC64FSZ + STKB + TF_L + (4*8)], %l4 2682 ldx [%sp + CC64FSZ + STKB + TF_L + (5*8)], %l5 2683 ldx [%sp + CC64FSZ + STKB + TF_L + (6*8)], %l6 2684 ldx [%sp + CC64FSZ + STKB + TF_L + (7*8)], %l7 2685 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)] 2686 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)] 2687 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)] 2688 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)] 2689 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)] 2690 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)] 2691 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)] 2692 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)] 2693#endif 2694 ba,pt %xcc, Lslowtrap_reenter 2695 mov %g6, %sp 2696#endif 2697 2698#if 0 2699/* 2700 * breakpoint: capture as much info as possible and then call DDB 2701 * or trap, as the case may be. 2702 * 2703 * First, we switch to interrupt globals, and blow away %g7. Then 2704 * switch down one stackframe -- just fiddle w/cwp, don't save or 2705 * we'll trap. Then slowly save all the globals into our static 2706 * register buffer. etc. etc. 2707 */ 2708 2709breakpoint: 2710 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! Get IG to use 2711 rdpr %cwp, %g7 2712 inc 1, %g7 ! Equivalent of save 2713 wrpr %g7, 0, %cwp ! Now we have some unused locals to fiddle with 2714XXX ddb_regs is now ddb-regp and is a pointer not a symbol. 2715 set _C_LABEL(ddb_regs), %l0 2716 stx %g1, [%l0+DBR_IG+(1*8)] ! Save IGs 2717 stx %g2, [%l0+DBR_IG+(2*8)] 2718 stx %g3, [%l0+DBR_IG+(3*8)] 2719 stx %g4, [%l0+DBR_IG+(4*8)] 2720 stx %g5, [%l0+DBR_IG+(5*8)] 2721 stx %g6, [%l0+DBR_IG+(6*8)] 2722 stx %g7, [%l0+DBR_IG+(7*8)] 2723 wrpr %g0, PSTATE_KERN|PSTATE_MG, %pstate ! Get MG to use 2724 stx %g1, [%l0+DBR_MG+(1*8)] ! Save MGs 2725 stx %g2, [%l0+DBR_MG+(2*8)] 2726 stx %g3, [%l0+DBR_MG+(3*8)] 2727 stx %g4, [%l0+DBR_MG+(4*8)] 2728 stx %g5, [%l0+DBR_MG+(5*8)] 2729 stx %g6, [%l0+DBR_MG+(6*8)] 2730 stx %g7, [%l0+DBR_MG+(7*8)] 2731 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Get AG to use 2732 stx %g1, [%l0+DBR_AG+(1*8)] ! Save AGs 2733 stx %g2, [%l0+DBR_AG+(2*8)] 2734 stx %g3, [%l0+DBR_AG+(3*8)] 2735 stx %g4, [%l0+DBR_AG+(4*8)] 2736 stx %g5, [%l0+DBR_AG+(5*8)] 2737 stx %g6, [%l0+DBR_AG+(6*8)] 2738 stx %g7, [%l0+DBR_AG+(7*8)] 2739 wrpr %g0, PSTATE_KERN, %pstate ! Get G to use 2740 stx %g1, [%l0+DBR_G+(1*8)] ! Save Gs 2741 stx %g2, [%l0+DBR_G+(2*8)] 2742 stx %g3, [%l0+DBR_G+(3*8)] 2743 stx %g4, [%l0+DBR_G+(4*8)] 2744 stx %g5, [%l0+DBR_G+(5*8)] 2745 stx %g6, [%l0+DBR_G+(6*8)] 2746 stx %g7, [%l0+DBR_G+(7*8)] 2747 rdpr %canrestore, %l1 2748 stb %l1, [%l0+DBR_CANRESTORE] 2749 rdpr %cansave, %l2 2750 stb %l2, [%l0+DBR_CANSAVE] 2751 rdpr %cleanwin, %l3 2752 stb %l3, [%l0+DBR_CLEANWIN] 2753 rdpr %wstate, %l4 2754 stb %l4, [%l0+DBR_WSTATE] 2755 rd %y, %l5 2756 stw %l5, [%l0+DBR_Y] 2757 rdpr %tl, %l6 2758 stb %l6, [%l0+DBR_TL] 2759 dec 1, %g7 2760#endif 2761 2762/* 2763 * I will not touch any of the DDB or KGDB stuff until I know what's going 2764 * on with the symbol table. This is all still v7/v8 code and needs to be fixed. 2765 */ 2766#ifdef KGDB 2767/* 2768 * bpt is entered on all breakpoint traps. 2769 * If this is a kernel breakpoint, we do not want to call trap(). 2770 * Among other reasons, this way we can set breakpoints in trap(). 2771 */ 2772bpt: 2773 set TSTATE_PRIV, %l4 2774 andcc %l4, %l0, %g0 ! breakpoint from kernel? 2775 bz slowtrap ! no, go do regular trap 2776 nop 2777 2778 /* 2779 * Build a trap frame for kgdb_trap_glue to copy. 2780 * Enable traps but set ipl high so that we will not 2781 * see interrupts from within breakpoints. 2782 */ 2783 save %sp, -CCFSZ-TF_SIZE, %sp ! allocate a trap frame 2784 TRAP_SETUP(-CCFSZ-TF_SIZE) 2785 or %l0, PSR_PIL, %l4 ! splhigh() 2786 wr %l4, 0, %psr ! the manual claims that this 2787 wr %l4, PSR_ET, %psr ! song and dance is necessary 2788 std %l0, [%sp + CCFSZ + 0] ! tf.tf_psr, tf.tf_pc 2789 mov %l3, %o0 ! trap type arg for kgdb_trap_glue 2790 rd %y, %l3 2791 std %l2, [%sp + CCFSZ + 8] ! tf.tf_npc, tf.tf_y 2792 rd %wim, %l3 2793 st %l3, [%sp + CCFSZ + 16] ! tf.tf_wim (a kgdb-only r/o field) 2794 st %g1, [%sp + CCFSZ + 20] ! tf.tf_global[1] 2795 std %g2, [%sp + CCFSZ + 24] ! etc 2796 std %g4, [%sp + CCFSZ + 32] 2797 std %g6, [%sp + CCFSZ + 40] 2798 std %i0, [%sp + CCFSZ + 48] ! tf.tf_in[0..1] 2799 std %i2, [%sp + CCFSZ + 56] ! etc 2800 std %i4, [%sp + CCFSZ + 64] 2801 std %i6, [%sp + CCFSZ + 72] 2802 2803 /* 2804 * Now call kgdb_trap_glue(); if it returns, call trap(). 2805 */ 2806 mov %o0, %l3 ! gotta save trap type 2807 call _C_LABEL(kgdb_trap_glue) ! kgdb_trap_glue(type, &trapframe) 2808 add %sp, CCFSZ, %o1 ! (&trapframe) 2809 2810 /* 2811 * Use slowtrap to call trap---but first erase our tracks 2812 * (put the registers back the way they were). 2813 */ 2814 mov %l3, %o0 ! slowtrap will need trap type 2815 ld [%sp + CCFSZ + 12], %l3 2816 wr %l3, 0, %y 2817 ld [%sp + CCFSZ + 20], %g1 2818 ldd [%sp + CCFSZ + 24], %g2 2819 ldd [%sp + CCFSZ + 32], %g4 2820 b Lslowtrap_reenter 2821 ldd [%sp + CCFSZ + 40], %g6 2822 2823/* 2824 * Enter kernel breakpoint. Write all the windows (not including the 2825 * current window) into the stack, so that backtrace works. Copy the 2826 * supplied trap frame to the kgdb stack and switch stacks. 2827 * 2828 * kgdb_trap_glue(type, tf0) 2829 * int type; 2830 * struct trapframe *tf0; 2831 */ 2832ENTRY_NOPROFILE(kgdb_trap_glue) 2833 save %sp, -CCFSZ, %sp 2834 2835 flushw ! flush all windows 2836 mov %sp, %l4 ! %l4 = current %sp 2837 2838 /* copy trapframe to top of kgdb stack */ 2839 set _C_LABEL(kgdb_stack) + KGDB_STACK_SIZE - 80, %l0 2840 ! %l0 = tfcopy -> end_of_kgdb_stack 2841 mov 80, %l1 28421: ldd [%i1], %l2 2843 inc 8, %i1 2844 deccc 8, %l1 2845 std %l2, [%l0] 2846 bg 1b 2847 inc 8, %l0 2848 2849#ifdef NOTDEF_DEBUG 2850 /* save old red zone and then turn it off */ 2851 sethi %hi(_C_LABEL(redzone)), %l7 2852 ld [%l7 + %lo(_C_LABEL(redzone))], %l6 2853 st %g0, [%l7 + %lo(_C_LABEL(redzone))] 2854#endif 2855 /* switch to kgdb stack */ 2856 add %l0, -CCFSZ-TF_SIZE, %sp 2857 2858 /* if (kgdb_trap(type, tfcopy)) kgdb_rett(tfcopy); */ 2859 mov %i0, %o0 2860 call _C_LABEL(kgdb_trap) 2861 add %l0, -80, %o1 2862 tst %o0 2863 bnz,a kgdb_rett 2864 add %l0, -80, %g1 2865 2866 /* 2867 * kgdb_trap() did not handle the trap at all so the stack is 2868 * still intact. A simple `restore' will put everything back, 2869 * after we reset the stack pointer. 2870 */ 2871 mov %l4, %sp 2872#ifdef NOTDEF_DEBUG 2873 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! restore red zone 2874#endif 2875 ret 2876 restore 2877 2878/* 2879 * Return from kgdb trap. This is sort of special. 2880 * 2881 * We know that kgdb_trap_glue wrote the window above it, so that we will 2882 * be able to (and are sure to have to) load it up. We also know that we 2883 * came from kernel land and can assume that the %fp (%i6) we load here 2884 * is proper. We must also be sure not to lower ipl (it is at splhigh()) 2885 * until we have traps disabled, due to the SPARC taking traps at the 2886 * new ipl before noticing that PSR_ET has been turned off. We are on 2887 * the kgdb stack, so this could be disastrous. 2888 * 2889 * Note that the trapframe argument in %g1 points into the current stack 2890 * frame (current window). We abandon this window when we move %g1->tf_psr 2891 * into %psr, but we will not have loaded the new %sp yet, so again traps 2892 * must be disabled. 2893 */ 2894kgdb_rett: 2895 rd %psr, %g4 ! turn off traps 2896 wr %g4, PSR_ET, %psr 2897 /* use the three-instruction delay to do something useful */ 2898 ld [%g1], %g2 ! pick up new %psr 2899 ld [%g1 + 12], %g3 ! set %y 2900 wr %g3, 0, %y 2901#ifdef NOTDEF_DEBUG 2902 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! and restore red zone 2903#endif 2904 wr %g0, 0, %wim ! enable window changes 2905 nop; nop; nop 2906 /* now safe to set the new psr (changes CWP, leaves traps disabled) */ 2907 wr %g2, 0, %psr ! set rett psr (including cond codes) 2908 /* 3 instruction delay before we can use the new window */ 2909/*1*/ ldd [%g1 + 24], %g2 ! set new %g2, %g3 2910/*2*/ ldd [%g1 + 32], %g4 ! set new %g4, %g5 2911/*3*/ ldd [%g1 + 40], %g6 ! set new %g6, %g7 2912 2913 /* now we can use the new window */ 2914 mov %g1, %l4 2915 ld [%l4 + 4], %l1 ! get new pc 2916 ld [%l4 + 8], %l2 ! get new npc 2917 ld [%l4 + 20], %g1 ! set new %g1 2918 2919 /* set up returnee's out registers, including its %sp */ 2920 ldd [%l4 + 48], %i0 2921 ldd [%l4 + 56], %i2 2922 ldd [%l4 + 64], %i4 2923 ldd [%l4 + 72], %i6 2924 2925 /* load returnee's window, making the window above it be invalid */ 2926 restore 2927 restore %g0, 1, %l1 ! move to inval window and set %l1 = 1 2928 rd %psr, %l0 2929 srl %l1, %l0, %l1 2930 wr %l1, 0, %wim ! %wim = 1 << (%psr & 31) 2931 sethi %hi(CPCB), %l1 2932 LDPTR [%l1 + %lo(CPCB)], %l1 2933 and %l0, 31, %l0 ! CWP = %psr & 31; 2934! st %l0, [%l1 + PCB_WIM] ! cpcb->pcb_wim = CWP; 2935 save %g0, %g0, %g0 ! back to window to reload 2936! LOADWIN(%sp) 2937 save %g0, %g0, %g0 ! back to trap window 2938 /* note, we have not altered condition codes; safe to just rett */ 2939 RETT 2940#endif 2941 2942/* 2943 * syscall_setup() builds a trap frame and calls syscall(). 2944 * sun_syscall is same but delivers sun system call number 2945 * XXX should not have to save&reload ALL the registers just for 2946 * ptrace... 2947 */ 2948syscall_setup: 2949#ifdef TRAPS_USE_IG 2950 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 2951#endif 2952 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2953 2954#ifdef DEBUG 2955 rdpr %tt, %o1 ! debug 2956 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 2957#endif 2958 2959 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 2960 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 2961 mov %g1, %o1 ! code 2962 rdpr %tpc, %o2 ! (pc) 2963 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 2964 rdpr %tstate, %g1 2965 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 2966 rdpr %tnpc, %o3 2967 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 2968 rd %y, %o4 2969 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 2970 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 2971 wrpr %g0, 0, %tl ! return to tl=0 2972 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 2973 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 2974 2975 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 2976 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 2977 stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] 2978 st %o4, [%sp + CC64FSZ + STKB + TF_Y] 2979 2980 rdpr %pil, %g5 2981 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 2982 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2983 2984 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2985 !! In our case we need to clear it before calling any C-code 2986 clr %g4 2987 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2988 2989 sethi %hi(CURLWP), %l1 2990 LDPTR [%l1 + %lo(CURLWP)], %l1 2991 LDPTR [%l1 + L_PROC], %l1 ! now %l1 points to p 2992 LDPTR [%l1 + P_MD_SYSCALL], %l1 2993 call %l1 2994 wrpr %g0, PSTATE_INTR, %pstate ! turn on interrupts 2995 2996 /* see `lwp_trampoline' for the reason for this label */ 2997return_from_syscall: 2998 wrpr %g0, PSTATE_KERN, %pstate ! Disable intterrupts 2999 wrpr %g0, 0, %tl ! Return to tl==0 3000 ba,a,pt %icc, return_from_trap 3001 nop 3002 NOTREACHED 3003 3004/* 3005 * interrupt_vector: 3006 * 3007 * Spitfire chips never get level interrupts directly from H/W. 3008 * Instead, all interrupts come in as interrupt_vector traps. 3009 * The interrupt number or handler address is an 11 bit number 3010 * encoded in the first interrupt data word. Additional words 3011 * are application specific and used primarily for cross-calls. 3012 * 3013 * The interrupt vector handler then needs to identify the 3014 * interrupt source from the interrupt number and arrange to 3015 * invoke the interrupt handler. This can either be done directly 3016 * from here, or a softint at a particular level can be issued. 3017 * 3018 * To call an interrupt directly and not overflow the trap stack, 3019 * the trap registers should be saved on the stack, registers 3020 * cleaned, trap-level decremented, the handler called, and then 3021 * the process must be reversed. 3022 * 3023 * To simplify life all we do here is issue an appropriate softint. 3024 * 3025 * Note: It is impossible to identify or change a device's 3026 * interrupt number until it is probed. That's the 3027 * purpose for all the funny interrupt acknowledge 3028 * code. 3029 * 3030 */ 3031 3032/* 3033 * Vectored interrupts: 3034 * 3035 * When an interrupt comes in, interrupt_vector uses the interrupt 3036 * vector number to lookup the appropriate intrhand from the intrlev 3037 * array. It then looks up the interrupt level from the intrhand 3038 * structure. It uses the level to index the intrpending array, 3039 * which is 8 slots for each possible interrupt level (so we can 3040 * shift instead of multiply for address calculation). It hunts for 3041 * any available slot at that level. Available slots are NULL. 3042 * 3043 * Then interrupt_vector uses the interrupt level in the intrhand 3044 * to issue a softint of the appropriate level. The softint handler 3045 * figures out what level interrupt it's handling and pulls the first 3046 * intrhand pointer out of the intrpending array for that interrupt 3047 * level, puts a NULL in its place, clears the interrupt generator, 3048 * and invokes the interrupt handler. 3049 */ 3050 3051/* intrpending array is now in per-CPU structure. */ 3052 3053#ifdef DEBUG 3054#define INTRDEBUG_VECTOR 0x1 3055#define INTRDEBUG_LEVEL 0x2 3056#define INTRDEBUG_FUNC 0x4 3057#define INTRDEBUG_SPUR 0x8 3058 .data 3059 .globl _C_LABEL(intrdebug) 3060_C_LABEL(intrdebug): .word 0x0 3061/* 3062 * Note: we use the local label `97' to branch forward to, to skip 3063 * actual debugging code following a `intrdebug' bit test. 3064 */ 3065#endif 3066 .text 3067interrupt_vector: 3068#ifdef TRAPSTATS 3069 set _C_LABEL(kiveccnt), %g1 3070 set _C_LABEL(iveccnt), %g2 3071 rdpr %tl, %g3 3072 dec %g3 3073 movrz %g3, %g2, %g1 3074 lduw [%g1], %g2 3075 inc %g2 3076 stw %g2, [%g1] 3077#endif 3078 ldxa [%g0] ASI_IRSR, %g1 3079 mov IRDR_0H, %g7 3080 ldxa [%g7] ASI_IRDR, %g7 ! Get interrupt number 3081 membar #Sync 3082 3083 btst IRSR_BUSY, %g1 3084 bz,pn %icc, 3f ! spurious interrupt 3085#ifdef MULTIPROCESSOR 3086 sethi %hi(KERNBASE), %g1 3087 3088 cmp %g7, %g1 3089 bl,pt %xcc, Lsoftint_regular ! >= KERNBASE is a fast cross-call 3090 cmp %g7, MAXINTNUM 3091 3092 mov IRDR_1H, %g2 3093 ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument 1 3094 mov IRDR_2H, %g3 3095 ldxa [%g3] ASI_IRDR, %g3 ! Get IPI handler argument 2 3096 3097 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 3098 membar #Sync ! Should not be needed due to retry 3099 3100 jmpl %g7, %g0 3101 nop 3102#else 3103 cmp %g7, MAXINTNUM 3104#endif 3105 3106Lsoftint_regular: 3107 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 3108 membar #Sync ! Should not be needed due to retry 3109 sllx %g7, PTRSHFT, %g5 ! Calculate entry number 3110 sethi %hi(_C_LABEL(intrlev)), %g3 3111 bgeu,pn %xcc, 3f 3112 or %g3, %lo(_C_LABEL(intrlev)), %g3 3113 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 3114 brz,pn %g5, 3f ! NULL means it isn't registered yet. Skip it. 3115 nop 3116 3117setup_sparcintr: 3118 LDPTR [%g5+IH_PEND], %g6 ! Read pending flag 3119 brnz,pn %g6, ret_from_intr_vector ! Skip it if it's running 3120 ldub [%g5+IH_PIL], %g6 ! Read interrupt mask 3121 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %g1 3122 sll %g6, PTRSHFT, %g3 ! Find start of table for this IPL 3123 or %g1, %lo(CPUINFO_VA+CI_INTRPENDING), %g1 3124 add %g1, %g3, %g1 31251: 3126 LDPTR [%g1], %g3 ! Load list head 3127 STPTR %g3, [%g5+IH_PEND] ! Link our intrhand node in 3128 mov %g5, %g7 3129 CASPTR [%g1] ASI_N, %g3, %g7 3130 cmp %g7, %g3 ! Did it work? 3131 bne,pn CCCR, 1b ! No, try again 3132 .empty 31332: 3134#ifdef NOT_DEBUG 3135 set _C_LABEL(intrdebug), %g7 3136 ld [%g7], %g7 3137 btst INTRDEBUG_VECTOR, %g7 3138 bz,pt %icc, 97f 3139 nop 3140 3141 cmp %g6, 0xa ! ignore clock interrupts? 3142 bz,pt %icc, 97f 3143 nop 3144 3145 STACKFRAME(-CC64FSZ) ! Get a clean register window 3146 LOAD_ASCIZ(%o0,\ 3147 "interrupt_vector: number %lx softint mask %lx pil %lu slot %p\r\n") 3148 mov %g2, %o1 3149 rdpr %pil, %o3 3150 mov %g1, %o4 3151 GLOBTOLOC 3152 clr %g4 3153 call prom_printf 3154 mov %g6, %o2 3155 LOCTOGLOB 3156 restore 315797: 3158#endif 3159 mov 1, %g7 3160 sll %g7, %g6, %g6 3161 wr %g6, 0, SET_SOFTINT ! Invoke a softint 3162 3163 .global ret_from_intr_vector 3164ret_from_intr_vector: 3165 retry 3166 NOTREACHED 3167 31683: 3169#ifdef NOT_DEBUG /* always do this */ 3170 set _C_LABEL(intrdebug), %g6 3171 ld [%g6], %g6 3172 btst INTRDEBUG_SPUR, %g6 3173 bz,pt %icc, 97f 3174 nop 3175#endif 3176#if 1 3177 STACKFRAME(-CC64FSZ) ! Get a clean register window 3178 LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\r\n") 3179 mov %g7, %o1 3180 GLOBTOLOC 3181 clr %g4 3182 call prom_printf 3183 rdpr %pil, %o2 3184 LOCTOGLOB 3185 restore 318697: 3187#endif 3188 ba,a ret_from_intr_vector 3189 nop ! XXX spitfire bug? 3190 3191/* 3192 * Ultra1 and Ultra2 CPUs use soft interrupts for everything. What we do 3193 * on a soft interrupt, is we should check which bits in ASR_SOFTINT(0x16) 3194 * are set, handle those interrupts, then clear them by setting the 3195 * appropriate bits in ASR_CLEAR_SOFTINT(0x15). 3196 * 3197 * We have an array of 8 interrupt vector slots for each of 15 interrupt 3198 * levels. If a vectored interrupt can be dispatched, the dispatch 3199 * routine will place a pointer to an intrhand structure in one of 3200 * the slots. The interrupt handler will go through the list to look 3201 * for an interrupt to dispatch. If it finds one it will pull it off 3202 * the list, free the entry, and call the handler. The code is like 3203 * this: 3204 * 3205 * for (i=0; i<8; i++) 3206 * if (ih = intrpending[intlev][i]) { 3207 * intrpending[intlev][i] = NULL; 3208 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 3209 * return; 3210 * strayintr(&frame); 3211 * return; 3212 * } 3213 * 3214 * Otherwise we go back to the old style of polled interrupts. 3215 * 3216 * After preliminary setup work, the interrupt is passed to each 3217 * registered handler in turn. These are expected to return nonzero if 3218 * they took care of the interrupt. If a handler claims the interrupt, 3219 * we exit (hardware interrupts are latched in the requestor so we'll 3220 * just take another interrupt in the unlikely event of simultaneous 3221 * interrupts from two different devices at the same level). If we go 3222 * through all the registered handlers and no one claims it, we report a 3223 * stray interrupt. This is more or less done as: 3224 * 3225 * for (ih = intrhand[intlev]; ih; ih = ih->ih_next) 3226 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 3227 * return; 3228 * strayintr(&frame); 3229 * 3230 * Inputs: 3231 * %l0 = %tstate 3232 * %l1 = return pc 3233 * %l2 = return npc 3234 * %l3 = interrupt level 3235 * (software interrupt only) %l4 = bits to clear in interrupt register 3236 * 3237 * Internal: 3238 * %l4, %l5: local variables 3239 * %l6 = %y 3240 * %l7 = %g1 3241 * %g2..%g7 go to stack 3242 * 3243 * An interrupt frame is built in the space for a full trapframe; 3244 * this contains the psr, pc, npc, and interrupt level. 3245 * 3246 * The level of this interrupt is determined by: 3247 * 3248 * IRQ# = %tt - 0x40 3249 */ 3250 3251ENTRY_NOPROFILE(sparc_interrupt) 3252#ifdef TRAPS_USE_IG 3253 ! This is for interrupt debugging 3254 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 3255#endif 3256 /* 3257 * If this is a %tick softint, clear it then call interrupt_vector. 3258 */ 3259 rd SOFTINT, %g1 3260 btst 1, %g1 3261 bz,pt %icc, 0f 3262 sethi %hi(CPUINFO_VA+CI_TICK_IH), %g3 3263 wr %g0, 1, CLEAR_SOFTINT 3264 ba,pt %icc, setup_sparcintr 3265 LDPTR [%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5 32660: 3267 3268 ! Increment the per-cpu interrupt level 3269 sethi %hi(CPUINFO_VA+CI_IDEPTH), %g1 3270 ld [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)], %g2 3271 inc %g2 3272 st %g2, [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)] 3273 3274#ifdef TRAPSTATS 3275 sethi %hi(_C_LABEL(kintrcnt)), %g1 3276 sethi %hi(_C_LABEL(uintrcnt)), %g2 3277 or %g1, %lo(_C_LABEL(kintrcnt)), %g1 3278 or %g1, %lo(_C_LABEL(uintrcnt)), %g2 3279 rdpr %tl, %g3 3280 dec %g3 3281 movrz %g3, %g2, %g1 3282 lduw [%g1], %g2 3283 inc %g2 3284 stw %g2, [%g1] 3285 /* See if we're on the interrupt stack already. */ 3286 set EINTSTACK, %g2 3287 set (EINTSTACK-INTSTACK), %g1 3288 btst 1, %sp 3289 add %sp, BIAS, %g3 3290 movz %icc, %sp, %g3 3291 srl %g3, 0, %g3 3292 sub %g2, %g3, %g3 3293 cmp %g3, %g1 3294 bgu 1f 3295 set _C_LABEL(intristk), %g1 3296 lduw [%g1], %g2 3297 inc %g2 3298 stw %g2, [%g1] 32991: 3300#endif 3301 INTR_SETUP(-CC64FSZ-TF_SIZE) 3302 ! Switch to normal globals so we can save them 3303 wrpr %g0, PSTATE_KERN, %pstate 3304 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 3305 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 3306 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 3307 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 3308 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 3309 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 3310 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 3311 3312 /* 3313 * In the EMBEDANY memory model %g4 points to the start of the 3314 * data segment. In our case we need to clear it before calling 3315 * any C-code. 3316 */ 3317 clr %g4 3318 3319 flushw ! Do not remove this insn -- causes interrupt loss 3320 rd %y, %l6 3321 INCR64(CPUINFO_VA+CI_NINTR) ! cnt.v_ints++ (clobbers %o0,%o1) 3322 rdpr %tt, %l5 ! Find out our current IPL 3323 rdpr %tstate, %l0 3324 rdpr %tpc, %l1 3325 rdpr %tnpc, %l2 3326 rdpr %tl, %l3 ! Dump our trap frame now we have taken the IRQ 3327 stw %l6, [%sp + CC64FSZ + STKB + TF_Y] ! Silly, but we need to save this for rft 3328 dec %l3 3329 wrpr %g0, %l3, %tl 3330 sth %l5, [%sp + CC64FSZ + STKB + TF_TT]! debug 3331 stx %l0, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set up intrframe/clockframe 3332 stx %l1, [%sp + CC64FSZ + STKB + TF_PC] 3333 btst TSTATE_PRIV, %l0 ! User mode? 3334 stx %l2, [%sp + CC64FSZ + STKB + TF_NPC] 3335 3336 sub %l5, 0x40, %l6 ! Convert to interrupt level 3337 sethi %hi(_C_LABEL(intr_evcnts)), %l4 3338 stb %l6, [%sp + CC64FSZ + STKB + TF_PIL] ! set up intrframe/clockframe 3339 rdpr %pil, %o1 3340 mulx %l6, EVC_SIZE, %l3 3341 or %l4, %lo(_C_LABEL(intr_evcnts)), %l4 ! intrcnt[intlev]++; 3342 stb %o1, [%sp + CC64FSZ + STKB + TF_OLDPIL] ! old %pil 3343 ldx [%l4 + %l3], %o0 3344 add %l4, %l3, %l4 3345 clr %l5 ! Zero handled count 3346#ifdef MULTIPROCESSOR 3347 mov 1, %l3 ! Ack softint 33481: add %o0, 1, %l7 3349 casxa [%l4] ASI_N, %o0, %l7 3350 cmp %o0, %l7 3351 bne,a,pn %xcc, 1b ! retry if changed 3352 mov %l7, %o0 3353#else 3354 inc %o0 3355 mov 1, %l3 ! Ack softint 3356 stx %o0, [%l4] 3357#endif 3358 sll %l3, %l6, %l3 ! Generate IRQ mask 3359 3360 wrpr %l6, %pil 3361 3362sparc_intr_retry: 3363 wr %l3, 0, CLEAR_SOFTINT ! (don't clear possible %tick IRQ) 3364 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %l4 3365 sll %l6, PTRSHFT, %l2 3366 or %l4, %lo(CPUINFO_VA+CI_INTRPENDING), %l4 3367 add %l2, %l4, %l4 3368 33691: 3370 membar #StoreLoad ! Make sure any failed casxa insns complete 3371 LDPTR [%l4], %l2 ! Check a slot 3372 cmp %l2, -1 3373 beq,pn CCCR, intrcmplt ! Empty list? 3374 mov -1, %l7 3375 membar #LoadStore 3376 CASPTR [%l4] ASI_N, %l2, %l7 ! Grab the entire list 3377 cmp %l7, %l2 3378 bne,pn CCCR, 1b 3379 .empty 33802: 3381 add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB 3382 LDPTR [%l2 + IH_PEND], %l7 ! save ih->ih_pending 3383 membar #LoadStore 3384 STPTR %g0, [%l2 + IH_PEND] ! Clear pending flag 3385 membar #Sync 3386 LDPTR [%l2 + IH_FUN], %o4 ! ih->ih_fun 3387 LDPTR [%l2 + IH_ARG], %o0 ! ih->ih_arg 3388 3389#ifdef NOT_DEBUG 3390 set _C_LABEL(intrdebug), %o3 3391 ld [%o2], %o3 3392 btst INTRDEBUG_FUNC, %o3 3393 bz,a,pt %icc, 97f 3394 nop 3395 3396 cmp %l6, 0xa ! ignore clock interrupts? 3397 bz,pt %icc, 97f 3398 nop 3399 3400 STACKFRAME(-CC64FSZ) ! Get a clean register window 3401 LOAD_ASCIZ(%o0, "sparc_interrupt: func %p arg %p\r\n") 3402 mov %i0, %o2 ! arg 3403 GLOBTOLOC 3404 call prom_printf 3405 mov %i4, %o1 ! func 3406 LOCTOGLOB 3407 restore 340897: 3409 mov %l4, %o1 3410#endif 3411 3412 wrpr %g0, PSTATE_INTR, %pstate ! Reenable interrupts 3413 jmpl %o4, %o7 ! handled = (*ih->ih_fun)(...) 3414 movrz %o0, %o2, %o0 ! arg = (arg == 0) ? arg : tf 3415 wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts 3416 LDPTR [%l2 + IH_CLR], %l1 3417 membar #Sync 3418 3419 brz,pn %l1, 0f 3420 add %l5, %o0, %l5 3421 stx %g0, [%l1] ! Clear intr source 3422 membar #Sync ! Should not be needed 34230: 3424 cmp %l7, -1 3425 bne,pn CCCR, 2b ! 'Nother? 3426 mov %l7, %l2 3427 3428intrcmplt: 3429 /* 3430 * Re-read SOFTINT to see if any new pending interrupts 3431 * at this level. 3432 */ 3433 mov 1, %l3 ! Ack softint 3434 rd SOFTINT, %l7 ! %l5 contains #intr handled. 3435 sll %l3, %l6, %l3 ! Generate IRQ mask 3436 btst %l3, %l7 ! leave mask in %l3 for retry code 3437 bnz,pn %icc, sparc_intr_retry 3438 mov 1, %l5 ! initialize intr count for next run 3439 3440 ! Decrement this cpu's interrupt depth 3441 sethi %hi(CPUINFO_VA+CI_IDEPTH), %l4 3442 ld [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5 3443 dec %l5 3444 st %l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)] 3445 3446#ifdef NOT_DEBUG 3447 set _C_LABEL(intrdebug), %o2 3448 ld [%o2], %o2 3449 btst INTRDEBUG_FUNC, %o2 3450 bz,a,pt %icc, 97f 3451 nop 3452 3453 cmp %l6, 0xa ! ignore clock interrupts? 3454 bz,pt %icc, 97f 3455 nop 3456 3457 STACKFRAME(-CC64FSZ) ! Get a clean register window 3458 LOAD_ASCIZ(%o0, "sparc_interrupt: done\r\n") 3459 GLOBTOLOC 3460 call prom_printf 3461 nop 3462 LOCTOGLOB 3463 restore 346497: 3465#endif 3466 3467 ldub [%sp + CC64FSZ + STKB + TF_OLDPIL], %l3 ! restore old %pil 3468 wrpr %l3, 0, %pil 3469 3470 ba,a,pt %icc, return_from_trap 3471 nop 3472 3473#ifdef notyet 3474/* 3475 * Level 12 (ZS serial) interrupt. Handle it quickly, schedule a 3476 * software interrupt, and get out. Do the software interrupt directly 3477 * if we would just take it on the way out. 3478 * 3479 * Input: 3480 * %l0 = %psr 3481 * %l1 = return pc 3482 * %l2 = return npc 3483 * Internal: 3484 * %l3 = zs device 3485 * %l4, %l5 = temporary 3486 * %l6 = rr3 (or temporary data) + 0x100 => need soft int 3487 * %l7 = zs soft status 3488 */ 3489zshard: 3490#endif /* notyet */ 3491 3492 .globl return_from_trap, rft_kernel, rft_user 3493 .globl softtrap, slowtrap 3494 3495/* 3496 * Various return-from-trap routines (see return_from_trap). 3497 */ 3498 3499/* 3500 * Return from trap. 3501 * registers are: 3502 * 3503 * [%sp + CC64FSZ + STKB] => trap frame 3504 * 3505 * We must load all global, out, and trap registers from the trap frame. 3506 * 3507 * If returning to kernel, we should be at the proper trap level because 3508 * we don't touch %tl. 3509 * 3510 * When returning to user mode, the trap level does not matter, as it 3511 * will be set explicitly. 3512 * 3513 * If we are returning to user code, we must: 3514 * 1. Check for register windows in the pcb that belong on the stack. 3515 * If there are any, reload them 3516 */ 3517return_from_trap: 3518#ifdef DEBUG 3519 !! Make sure we don't have pc == npc == 0 or we suck. 3520 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 3521 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 3522 orcc %g2, %g3, %g0 3523 tz %icc, 1 3524#endif 3525 3526 !! 3527 !! We'll make sure we flush our pcb here, rather than later. 3528 !! 3529 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 3530 btst TSTATE_PRIV, %g1 ! returning to userland? 3531 3532 !! 3533 !! Let all pending interrupts drain before returning to userland 3534 !! 3535 bnz,pn %icc, 1f ! Returning to userland? 3536 nop 3537 wrpr %g0, PSTATE_INTR, %pstate 3538 wrpr %g0, %g0, %pil ! Lower IPL 35391: 3540 wrpr %g0, PSTATE_KERN, %pstate ! Make sure we have normal globals & no IRQs 3541 3542 /* Restore normal globals */ 3543 ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 3544 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 3545 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 3546 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 3547 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 3548 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 3549 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 3550 /* Switch to alternate globals and load outs */ 3551 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate 3552#ifdef TRAPS_USE_IG 3553 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 3554#endif 3555 ldx [%sp + CC64FSZ + STKB + TF_O + (0*8)], %i0 3556 ldx [%sp + CC64FSZ + STKB + TF_O + (1*8)], %i1 3557 ldx [%sp + CC64FSZ + STKB + TF_O + (2*8)], %i2 3558 ldx [%sp + CC64FSZ + STKB + TF_O + (3*8)], %i3 3559 ldx [%sp + CC64FSZ + STKB + TF_O + (4*8)], %i4 3560 ldx [%sp + CC64FSZ + STKB + TF_O + (5*8)], %i5 3561 ldx [%sp + CC64FSZ + STKB + TF_O + (6*8)], %i6 3562 ldx [%sp + CC64FSZ + STKB + TF_O + (7*8)], %i7 3563 /* Now load trap registers into alternate globals */ 3564 ld [%sp + CC64FSZ + STKB + TF_Y], %g4 3565 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! load new values 3566 wr %g4, 0, %y 3567 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 3568 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 3569 3570#ifdef NOTDEF_DEBUG 3571 ldub [%sp + CC64FSZ + STKB + TF_PIL], %g5 ! restore %pil 3572 wrpr %g5, %pil ! DEBUG 3573#endif 3574 3575 /* Returning to user mode or kernel mode? */ 3576 btst TSTATE_PRIV, %g1 ! returning to userland? 3577 bz,pt %icc, rft_user 3578 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! first instr of rft_user 3579 3580/* 3581 * Return from trap, to kernel. 3582 * 3583 * We will assume, for the moment, that all kernel traps are properly stacked 3584 * in the trap registers, so all we have to do is insert the (possibly modified) 3585 * register values into the trap registers then do a retry. 3586 * 3587 */ 3588rft_kernel: 3589 rdpr %tl, %g4 ! Grab a set of trap registers 3590 inc %g4 3591 wrpr %g4, %g0, %tl 3592 wrpr %g3, 0, %tnpc 3593 wrpr %g2, 0, %tpc 3594 wrpr %g1, 0, %tstate 3595 restore 3596 rdpr %tstate, %g1 ! Since we may have trapped our regs may be toast 3597 rdpr %cwp, %g2 3598 andn %g1, CWP, %g1 3599 wrpr %g1, %g2, %tstate ! Put %cwp in %tstate 3600 CLRTT 3601#ifdef TRAPSTATS 3602 rdpr %tl, %g2 3603 set _C_LABEL(rftkcnt), %g1 3604 sllx %g2, 2, %g2 3605 add %g1, %g2, %g1 3606 lduw [%g1], %g2 3607 inc %g2 3608 stw %g2, [%g1] 3609#endif 3610#if 0 3611 wrpr %g0, 0, %cleanwin ! DEBUG 3612#endif 3613#if defined(DDB) && defined(MULTIPROCESSOR) 3614 set sparc64_ipi_pause_trap_point, %g1 3615 rdpr %tpc, %g2 3616 cmp %g1, %g2 3617 bne,pt %icc, 0f 3618 nop 3619 done 36200: 3621#endif 3622 retry 3623 NOTREACHED 3624/* 3625 * Return from trap, to user. Checks for scheduling trap (`ast') first; 3626 * will re-enter trap() if set. Note that we may have to switch from 3627 * the interrupt stack to the kernel stack in this case. 3628 * %g1 = %tstate 3629 * %g2 = return %pc 3630 * %g3 = return %npc 3631 * If returning to a valid window, just set psr and return. 3632 */ 3633 .data 3634rft_wcnt: .word 0 3635 .text 3636 3637rft_user: 3638! sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! (done above) 3639 lduw [%g7 + %lo(CPUINFO_VA+CI_WANT_AST)], %g7! want AST trap? 3640 brnz,pn %g7, softtrap ! yes, re-enter trap with type T_AST 3641 mov T_AST, %g4 3642 3643#ifdef NOTDEF_DEBUG 3644 sethi %hi(CPCB), %g4 3645 LDPTR [%g4 + %lo(CPCB)], %g4 3646 ldub [%g4 + PCB_NSAVED], %g4 ! nsaved 3647 brz,pt %g4, 2f ! Only print if nsaved <> 0 3648 nop 3649 3650 set 1f, %o0 3651 mov %g4, %o1 3652 mov %g2, %o2 ! pc 3653 wr %g0, ASI_DMMU, %asi ! restore the user context 3654 ldxa [CTX_SECONDARY] %asi, %o3 ! ctx 3655 GLOBTOLOC 3656 mov %g3, %o5 3657 call printf 3658 mov %i6, %o4 ! sp 3659! wrpr %g0, PSTATE_INTR, %pstate ! Allow IRQ service 3660! wrpr %g0, PSTATE_KERN, %pstate ! DenyIRQ service 3661 LOCTOGLOB 36621: 3663 .data 3664 .asciz "rft_user: nsaved=%x pc=%d ctx=%x sp=%x npc=%p\n" 3665 _ALIGN 3666 .text 3667#endif 3668 3669 /* 3670 * NB: only need to do this after a cache miss 3671 */ 3672#ifdef TRAPSTATS 3673 set _C_LABEL(rftucnt), %g6 3674 lduw [%g6], %g7 3675 inc %g7 3676 stw %g7, [%g6] 3677#endif 3678 /* 3679 * Now check to see if any regs are saved in the pcb and restore them. 3680 * 3681 * Here we need to undo the damage caused by switching to a kernel 3682 * stack. 3683 * 3684 * We will use alternate globals %g4..%g7 because %g1..%g3 are used 3685 * by the data fault trap handlers and we don't want possible conflict. 3686 */ 3687 3688 sethi %hi(CPCB), %g6 3689 rdpr %otherwin, %g7 ! restore register window controls 3690#ifdef DEBUG 3691 rdpr %canrestore, %g5 ! DEBUG 3692 tst %g5 ! DEBUG 3693 tnz %icc, 1; nop ! DEBUG 3694! mov %g0, %g5 ! There should be *NO* %canrestore 3695 add %g7, %g5, %g7 ! DEBUG 3696#endif 3697 wrpr %g0, %g7, %canrestore 3698 LDPTR [%g6 + %lo(CPCB)], %g6 3699 wrpr %g0, 0, %otherwin 3700 3701 ldub [%g6 + PCB_NSAVED], %g7 ! Any saved reg windows? 3702 wrpr %g0, WSTATE_USER, %wstate ! Need to know where our sp points 3703 3704#ifdef DEBUG 3705 set rft_wcnt, %g4 ! Keep track of all the windows we restored 3706 stw %g7, [%g4] 3707#endif 3708 3709 brz,pt %g7, 5f ! No saved reg wins 3710 nop 3711 dec %g7 ! We can do this now or later. Move to last entry 3712 3713#ifdef DEBUG 3714 rdpr %canrestore, %g4 ! DEBUG Make sure we've restored everything 3715 brnz,a,pn %g4, 0f ! DEBUG 3716 sir ! DEBUG we should NOT have any usable windows here 37170: ! DEBUG 3718 wrpr %g0, 5, %tl 3719#endif 3720 rdpr %otherwin, %g4 3721 sll %g7, 7, %g5 ! calculate ptr into rw64 array 8*16 == 128 or 7 bits 3722 brz,pt %g4, 6f ! We should not have any user windows left 3723 add %g5, %g6, %g5 3724 3725 set 1f, %o0 3726 mov %g7, %o1 3727 mov %g4, %o2 3728 call printf 3729 wrpr %g0, PSTATE_KERN, %pstate 3730 set 2f, %o0 3731 call panic 3732 nop 3733 NOTREACHED 3734 .data 37351: .asciz "pcb_nsaved=%x and otherwin=%x\n" 37362: .asciz "rft_user\n" 3737 _ALIGN 3738 .text 37396: 37403: 3741 restored ! Load in the window 3742 restore ! This should not trap! 3743 ldx [%g5 + PCB_RW + ( 0*8)], %l0 ! Load the window from the pcb 3744 ldx [%g5 + PCB_RW + ( 1*8)], %l1 3745 ldx [%g5 + PCB_RW + ( 2*8)], %l2 3746 ldx [%g5 + PCB_RW + ( 3*8)], %l3 3747 ldx [%g5 + PCB_RW + ( 4*8)], %l4 3748 ldx [%g5 + PCB_RW + ( 5*8)], %l5 3749 ldx [%g5 + PCB_RW + ( 6*8)], %l6 3750 ldx [%g5 + PCB_RW + ( 7*8)], %l7 3751 3752 ldx [%g5 + PCB_RW + ( 8*8)], %i0 3753 ldx [%g5 + PCB_RW + ( 9*8)], %i1 3754 ldx [%g5 + PCB_RW + (10*8)], %i2 3755 ldx [%g5 + PCB_RW + (11*8)], %i3 3756 ldx [%g5 + PCB_RW + (12*8)], %i4 3757 ldx [%g5 + PCB_RW + (13*8)], %i5 3758 ldx [%g5 + PCB_RW + (14*8)], %i6 3759 ldx [%g5 + PCB_RW + (15*8)], %i7 3760 3761#ifdef DEBUG 3762 stx %g0, [%g5 + PCB_RW + (14*8)] ! DEBUG mark that we've saved this one 3763#endif 3764 3765 cmp %g5, %g6 3766 bgu,pt %xcc, 3b ! Next one? 3767 dec 8*16, %g5 3768 3769 rdpr %ver, %g5 3770 stb %g0, [%g6 + PCB_NSAVED] ! Clear them out so we won't do this again 3771 and %g5, CWP, %g5 3772 add %g5, %g7, %g4 3773 dec 1, %g5 ! NWINDOWS-1-1 3774 wrpr %g5, 0, %cansave 3775 wrpr %g0, 0, %canrestore ! Make sure we have no freeloaders XXX 3776 wrpr %g0, WSTATE_USER, %wstate ! Save things to user space 3777 mov %g7, %g5 ! We already did one restore 37784: 3779 rdpr %canrestore, %g4 3780 inc %g4 3781 deccc %g5 3782 wrpr %g4, 0, %cleanwin ! Make *sure* we don't trap to cleanwin 3783 bge,a,pt %xcc, 4b ! return to starting regwin 3784 save %g0, %g0, %g0 ! This may force a datafault 3785 3786#ifdef DEBUG 3787 wrpr %g0, 0, %tl 3788#endif 3789#ifdef TRAPSTATS 3790 set _C_LABEL(rftuld), %g5 3791 lduw [%g5], %g4 3792 inc %g4 3793 stw %g4, [%g5] 3794#endif 3795 !! 3796 !! We can't take any save faults in here 'cause they will never be serviced 3797 !! 3798 3799#ifdef DEBUG 3800 sethi %hi(CPCB), %g5 3801 LDPTR [%g5 + %lo(CPCB)], %g5 3802 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 3803 tst %g5 3804 tnz %icc, 1; nop ! Debugger if we still have saved windows 3805 bne,a rft_user ! Try starting over again 3806 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 3807#endif 3808 /* 3809 * Set up our return trapframe so we can recover if we trap from here 3810 * on in. 3811 */ 3812 wrpr %g0, 1, %tl ! Set up the trap state 3813 wrpr %g2, 0, %tpc 3814 wrpr %g3, 0, %tnpc 3815 ba,pt %icc, 6f 3816 wrpr %g1, %g0, %tstate 3817 38185: 3819 /* 3820 * Set up our return trapframe so we can recover if we trap from here 3821 * on in. 3822 */ 3823 wrpr %g0, 1, %tl ! Set up the trap state 3824 wrpr %g2, 0, %tpc 3825 wrpr %g3, 0, %tnpc 3826 wrpr %g1, %g0, %tstate 3827 restore 38286: 3829 rdpr %canrestore, %g5 3830 wrpr %g5, 0, %cleanwin ! Force cleanup of kernel windows 3831 3832#ifdef NOTDEF_DEBUG 3833 ldx [%g6 + CC64FSZ + STKB + TF_L + (0*8)], %g5! DEBUG -- get proper value for %l0 3834 cmp %l0, %g5 3835 be,a,pt %icc, 1f 3836 nop 3837! sir ! WATCHDOG 3838 set badregs, %g1 ! Save the suspect regs 3839 stw %l0, [%g1+(4*0)] 3840 stw %l1, [%g1+(4*1)] 3841 stw %l2, [%g1+(4*2)] 3842 stw %l3, [%g1+(4*3)] 3843 stw %l4, [%g1+(4*4)] 3844 stw %l5, [%g1+(4*5)] 3845 stw %l6, [%g1+(4*6)] 3846 stw %l7, [%g1+(4*7)] 3847 stw %i0, [%g1+(4*8)+(4*0)] 3848 stw %i1, [%g1+(4*8)+(4*1)] 3849 stw %i2, [%g1+(4*8)+(4*2)] 3850 stw %i3, [%g1+(4*8)+(4*3)] 3851 stw %i4, [%g1+(4*8)+(4*4)] 3852 stw %i5, [%g1+(4*8)+(4*5)] 3853 stw %i6, [%g1+(4*8)+(4*6)] 3854 stw %i7, [%g1+(4*8)+(4*7)] 3855 save 3856 inc %g7 3857 wrpr %g7, 0, %otherwin 3858 wrpr %g0, 0, %canrestore 3859 wrpr %g0, WSTATE_KERN, %wstate ! Need to know where our sp points 3860 set rft_wcnt, %g4 ! Restore nsaved before trapping 3861 sethi %hi(CPCB), %g6 3862 LDPTR [%g6 + %lo(CPCB)], %g6 3863 lduw [%g4], %g4 3864 stb %g4, [%g6 + PCB_NSAVED] 3865 ta 1 3866 sir 3867 .data 3868badregs: 3869 .space 16*4 3870 .text 38711: 3872#endif 3873 3874 rdpr %tstate, %g1 3875 rdpr %cwp, %g7 ! Find our cur window 3876 andn %g1, CWP, %g1 ! Clear it from %tstate 3877 wrpr %g1, %g7, %tstate ! Set %tstate with %cwp 3878 3879 wr %g0, ASI_DMMU, %asi ! restore the user context 3880 ldxa [CTX_SECONDARY] %asi, %g4 3881 sethi %hi(KERNBASE), %g7 ! Should not be needed due to retry 3882 stxa %g4, [CTX_PRIMARY] %asi 3883 membar #Sync ! Should not be needed due to retry 3884 flush %g7 ! Should not be needed due to retry 3885 CLRTT 3886#ifdef TRAPSTATS 3887 set _C_LABEL(rftudone), %g1 3888 lduw [%g1], %g2 3889 inc %g2 3890 stw %g2, [%g1] 3891#endif 3892#ifdef DEBUG 3893 sethi %hi(CPCB), %g5 3894 LDPTR [%g5 + %lo(CPCB)], %g5 3895 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 3896 tst %g5 3897 tnz %icc, 1; nop ! Debugger if we still have saved windows! 3898#endif 3899 wrpr %g0, 0, %pil ! Enable all interrupts 3900 retry 3901 3902! exported end marker for kernel gdb 3903 .globl _C_LABEL(endtrapcode) 3904_C_LABEL(endtrapcode): 3905 3906/* 3907 * Kernel entry point. 3908 * 3909 * The contract between bootloader and kernel is: 3910 * 3911 * %o0 OpenFirmware entry point, to keep Sun's updaters happy 3912 * %o1 Address of boot information vector (see bootinfo.h) 3913 * %o2 Length of the vector, in bytes 3914 * %o3 OpenFirmware entry point, to mimic Sun bootloader behavior 3915 * %o4 OpenFirmware, to meet earlier NetBSD kernels expectations 3916 */ 3917 .align 8 3918start: 3919dostart: 3920 mov 1, %g1 3921 sllx %g1, 63, %g1 3922 wr %g1, TICK_CMPR ! XXXXXXX clear and disable %tick_cmpr for now 3923 /* 3924 * Startup. 3925 * 3926 * The Sun FCODE bootloader is nice and loads us where we want 3927 * to be. We have a full set of mappings already set up for us. 3928 * 3929 * I think we end up having an entire 16M allocated to us. 3930 * 3931 * We enter with the prom entry vector in %o0, dvec in %o1, 3932 * and the bootops vector in %o2. 3933 * 3934 * All we need to do is: 3935 * 3936 * 1: Save the prom vector 3937 * 3938 * 2: Create a decent stack for ourselves 3939 * 3940 * 3: Install the permanent 4MB kernel mapping 3941 * 3942 * 4: Call the C language initialization code 3943 * 3944 */ 3945 3946 /* 3947 * Set the psr into a known state: 3948 * Set supervisor mode, interrupt level >= 13, traps enabled 3949 */ 3950 wrpr %g0, 13, %pil 3951 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 3952 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 3953 3954 /* 3955 * Step 2: Set up a v8-like stack if we need to 3956 */ 3957 3958#ifdef _LP64 3959 btst 1, %sp 3960 bnz,pt %icc, 0f 3961 nop 3962 add %sp, -BIAS, %sp 3963#else 3964 btst 1, %sp 3965 bz,pt %icc, 0f 3966 nop 3967 add %sp, BIAS, %sp 3968#endif 39690: 3970 3971 call _C_LABEL(bootstrap) 3972 clr %g4 ! Clear data segment pointer 3973 3974/* 3975 * Initialize the boot CPU. Basically: 3976 * 3977 * Locate the cpu_info structure for this CPU. 3978 * Establish a locked mapping for interrupt stack. 3979 * Switch to the initial stack. 3980 * Call the routine passed in in cpu_info->ci_spinup 3981 */ 3982 3983#ifdef NO_VCACHE 3984#define TTE_DATABITS TTE_L|TTE_CP|TTE_P|TTE_W 3985#else 3986#define TTE_DATABITS TTE_L|TTE_CP|TTE_CV|TTE_P|TTE_W 3987#endif 3988 3989 3990ENTRY_NOPROFILE(cpu_initialize) /* for cosmetic reasons - nicer backtrace */ 3991 /* 3992 * Step 5: is no more. 3993 */ 3994 3995 /* 3996 * Step 6: hunt through cpus list and find the one that 3997 * matches our UPAID. 3998 */ 3999 sethi %hi(_C_LABEL(cpus)), %l1 4000 ldxa [%g0] ASI_MID_REG, %l2 4001 LDPTR [%l1 + %lo(_C_LABEL(cpus))], %l1 4002 srax %l2, 17, %l2 ! Isolate UPAID from CPU reg 4003 and %l2, 0x1f, %l2 40040: 4005 ld [%l1 + CI_UPAID], %l3 ! Load UPAID 4006 cmp %l3, %l2 ! Does it match? 4007 bne,a,pt %icc, 0b ! no 4008 LDPTR [%l1 + CI_NEXT], %l1 ! Load next cpu_info pointer 4009 4010 4011 /* 4012 * Get pointer to our cpu_info struct 4013 */ 4014 mov %l1, %l7 ! save cpu_info pointer 4015 ldx [%l1 + CI_PADDR], %l1 ! Load the interrupt stack's PA 4016 4017 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 4018 sllx %l2, 32, %l2 ! Shift it into place 4019 4020 mov -1, %l3 ! Create a nice mask 4021 sllx %l3, 43, %l4 ! Mask off high bits 4022 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 4023 4024 andn %l1, %l4, %l1 ! Mask the phys page number 4025 4026 or %l2, %l1, %l1 ! Now take care of the high bits 4027 or %l1, TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 4028 4029 !! 4030 !! Now, map in the interrupt stack as context==0 4031 !! 4032 set TLB_TAG_ACCESS, %l5 4033 set INTSTACK, %l0 4034 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 4035 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 4036 membar #Sync 4037 4038 !! Setup kernel stack (we rely on curlwp on this cpu 4039 !! being lwp0 here and it's uarea is mapped special 4040 !! and already accessible here) 4041 flushw 4042 LDPTR [%l7 + CI_CPCB], %l0 ! load PCB/uarea pointer 4043 set USPACE - TF_SIZE - CC64FSZ, %l1 4044 add %l1, %l0, %l0 4045#ifdef _LP64 4046 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 4047 sub %l0, BIAS, %l0 ! and biased 4048#endif 4049 mov %l0, %sp 4050 flushw 4051 4052#ifdef DEBUG 4053 set _C_LABEL(pmapdebug), %o1 4054 ld [%o1], %o1 4055 sethi %hi(0x40000), %o2 4056 btst %o2, %o1 4057 bz 0f 4058 4059 set 1f, %o0 ! Debug printf 4060 call _C_LABEL(prom_printf) 4061 .data 40621: 4063 .asciz "Setting trap base...\r\n" 4064 _ALIGN 4065 .text 40660: 4067#endif 4068 /* 4069 * Step 7: change the trap base register, and install our TSB pointers 4070 */ 4071 4072 /* 4073 * install our TSB pointers 4074 */ 4075 sethi %hi(_C_LABEL(tsbsize)), %l2 4076 sethi %hi(0x1fff), %l3 4077 sethi %hi(TSB), %l4 4078 LDPTR [%l7 + CI_TSB_DMMU], %l0 4079 LDPTR [%l7 + CI_TSB_IMMU], %l1 4080 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 4081 or %l3, %lo(0x1fff), %l3 4082 or %l4, %lo(TSB), %l4 4083 4084 andn %l0, %l3, %l0 ! Mask off size and split bits 4085 or %l0, %l2, %l0 ! Make a TSB pointer 4086 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 4087 4088 andn %l1, %l3, %l1 ! Mask off size and split bits 4089 or %l1, %l2, %l1 ! Make a TSB pointer 4090 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 4091 membar #Sync 4092 set 1f, %l1 4093 flush %l1 40941: 4095 4096 /* set trap table */ 4097 set _C_LABEL(trapbase), %l1 4098 call _C_LABEL(prom_set_trap_table) ! Now we should be running 100% from our handlers 4099 mov %l1, %o0 4100 wrpr %l1, 0, %tba ! Make sure the PROM didn't foul up. 4101 4102 /* 4103 * Switch to the kernel mode and run away. 4104 */ 4105 wrpr %g0, WSTATE_KERN, %wstate 4106 4107#ifdef DEBUG 4108 wrpr %g0, 1, %tl ! Debug -- start at tl==3 so we'll watchdog 4109 wrpr %g0, 0x1ff, %tt ! Debug -- clear out unused trap regs 4110 wrpr %g0, 0, %tpc 4111 wrpr %g0, 0, %tnpc 4112 wrpr %g0, 0, %tstate 4113 wrpr %g0, 0, %tl 4114#endif 4115 4116#ifdef DEBUG 4117 set _C_LABEL(pmapdebug), %o1 4118 ld [%o1], %o1 4119 sethi %hi(0x40000), %o2 4120 btst %o2, %o1 4121 bz 0f 4122 4123 LDPTR [%l7 + CI_SPINUP], %o1 4124 set 1f, %o0 ! Debug printf 4125 call _C_LABEL(prom_printf) 4126 mov %sp, %o2 4127 4128 .data 41291: 4130 .asciz "Calling startup routine %p with stack at %p...\r\n" 4131 _ALIGN 4132 .text 41330: 4134#endif 4135 /* 4136 * Call our startup routine. 4137 */ 4138 4139 LDPTR [%l7 + CI_SPINUP], %o1 4140 4141 call %o1 ! Call routine 4142 clr %o0 ! our frame arg is ignored 4143 4144 set 1f, %o0 ! Main should never come back here 4145 call _C_LABEL(panic) 4146 nop 4147 .data 41481: 4149 .asciz "main() returned\r\n" 4150 _ALIGN 4151 .text 4152 4153 .align 8 4154ENTRY(get_romtba) 4155 retl 4156 rdpr %tba, %o0 4157 4158#ifdef MULTIPROCESSOR 4159 /* 4160 * cpu_mp_startup is called with: 4161 * 4162 * %g2 = cpu_args 4163 */ 4164ENTRY(cpu_mp_startup) 4165 mov 1, %o0 4166 sllx %o0, 63, %o0 4167 wr %o0, TICK_CMPR ! XXXXXXX clear and disable %tick_cmpr for now 4168 wrpr %g0, 0, %cleanwin 4169 wrpr %g0, 0, %tl ! Make sure we're not in NUCLEUS mode 4170 wrpr %g0, WSTATE_KERN, %wstate 4171 wrpr %g0, PSTATE_KERN, %pstate 4172 flushw 4173 4174 /* 4175 * Get pointer to our cpu_info struct 4176 */ 4177 ldx [%g2 + CBA_CPUINFO], %l1 ! Load the interrupt stack's PA 4178 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 4179 sllx %l2, 32, %l2 ! Shift it into place 4180 mov -1, %l3 ! Create a nice mask 4181 sllx %l3, 43, %l4 ! Mask off high bits 4182 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 4183 andn %l1, %l4, %l1 ! Mask the phys page number 4184 or %l2, %l1, %l1 ! Now take care of the high bits 4185 or %l1, TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 4186 4187 /* 4188 * Now, map in the interrupt stack & cpu_info as context==0 4189 */ 4190 set TLB_TAG_ACCESS, %l5 4191 set INTSTACK, %l0 4192 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 4193 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 4194 4195 /* 4196 * Set 0 as primary context XXX 4197 */ 4198 mov CTX_PRIMARY, %o0 4199 stxa %g0, [%o0] ASI_DMMU 4200 membar #Sync 4201 4202 /* 4203 * Temporarily use the interrupt stack 4204 */ 4205#ifdef _LP64 4206 set ((EINTSTACK - CC64FSZ - TF_SIZE)) & ~0x0f - BIAS, %sp 4207#else 4208 set EINTSTACK - CC64FSZ - TF_SIZE, %sp 4209#endif 4210 set 1, %fp 4211 clr %i7 4212 4213 /* 4214 * install our TSB pointers 4215 */ 4216 sethi %hi(CPUINFO_VA+CI_TSB_DMMU), %l0 4217 sethi %hi(CPUINFO_VA+CI_TSB_IMMU), %l1 4218 sethi %hi(_C_LABEL(tsbsize)), %l2 4219 sethi %hi(0x1fff), %l3 4220 sethi %hi(TSB), %l4 4221 LDPTR [%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0 4222 LDPTR [%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1 4223 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 4224 or %l3, %lo(0x1fff), %l3 4225 or %l4, %lo(TSB), %l4 4226 4227 andn %l0, %l3, %l0 ! Mask off size and split bits 4228 or %l0, %l2, %l0 ! Make a TSB pointer 4229 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 4230 membar #Sync 4231 4232 andn %l1, %l3, %l1 ! Mask off size and split bits 4233 or %l1, %l2, %l1 ! Make a TSB pointer 4234 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 4235 membar #Sync 4236 set 1f, %o0 4237 flush %o0 42381: 4239 4240 /* set trap table */ 4241 set _C_LABEL(trapbase), %l1 4242 call _C_LABEL(prom_set_trap_table) 4243 mov %l1, %o0 4244 wrpr %l1, 0, %tba ! Make sure the PROM didn't 4245 ! foul up. 4246 /* 4247 * Use this CPUs idlelewp's uarea stack 4248 */ 4249 sethi %hi(CPUINFO_VA+CI_IDLELWP), %l0 4250 LDPTR [%l0 + %lo(CPUINFO_VA+CI_IDLELWP)], %l0 4251 set USPACE - TF_SIZE - CC64FSZ, %l1 4252 LDPTR [%l0 + L_PCB], %l0 4253 add %l0, %l1, %l0 4254#ifdef _LP64 4255 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 4256 sub %l0, BIAS, %l0 ! and biased 4257#endif 4258 mov %l0, %sp 4259 flushw 4260 4261 /* 4262 * Switch to the kernel mode and run away. 4263 */ 4264 wrpr %g0, 13, %pil 4265 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 4266 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 4267 4268 call _C_LABEL(cpu_hatch) 4269 clr %g4 4270 4271 b _C_LABEL(idle_loop) 4272 clr %o0 4273 4274 NOTREACHED 4275 4276 .globl cpu_mp_startup_end 4277cpu_mp_startup_end: 4278#endif 4279 4280/* 4281 * openfirmware(cell* param); 4282 * 4283 * OpenFirmware entry point 4284 * 4285 * If we're running in 32-bit mode we need to convert to a 64-bit stack 4286 * and 64-bit cells. The cells we'll allocate off the stack for simplicity. 4287 */ 4288 .align 8 4289ENTRY(openfirmware) 4290 sethi %hi(romp), %o4 4291 andcc %sp, 1, %g0 4292 bz,pt %icc, 1f 4293 LDPTR [%o4+%lo(romp)], %o4 ! v9 stack, just load the addr and callit 4294 save %sp, -CC64FSZ, %sp 4295 rdpr %pil, %i2 4296 mov PIL_HIGH, %i3 4297 cmp %i3, %i2 4298 movle %icc, %i2, %i3 4299 wrpr %g0, %i3, %pil 4300 mov %i0, %o0 4301 mov %g1, %l1 4302 mov %g2, %l2 4303 mov %g3, %l3 4304 mov %g4, %l4 4305 mov %g5, %l5 4306 mov %g6, %l6 4307 mov %g7, %l7 4308 rdpr %pstate, %l0 4309 jmpl %i4, %o7 4310#if !defined(_LP64) 4311 wrpr %g0, PSTATE_PROM, %pstate 4312#else 4313 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 4314#endif 4315 wrpr %l0, %g0, %pstate 4316 mov %l1, %g1 4317 mov %l2, %g2 4318 mov %l3, %g3 4319 mov %l4, %g4 4320 mov %l5, %g5 4321 mov %l6, %g6 4322 mov %l7, %g7 4323 wrpr %i2, 0, %pil 4324 ret 4325 restore %o0, %g0, %o0 4326 43271: ! v8 -- need to screw with stack & params 4328#ifdef NOTDEF_DEBUG 4329 mov %o7, %o5 4330 call globreg_check 4331 nop 4332 mov %o5, %o7 4333#endif 4334 save %sp, -CC64FSZ, %sp ! Get a new 64-bit stack frame 4335 add %sp, -BIAS, %sp 4336 rdpr %pstate, %l0 4337 srl %sp, 0, %sp 4338 rdpr %pil, %i2 ! s = splx(level) 4339 mov %i0, %o0 4340 mov PIL_HIGH, %i3 4341 mov %g1, %l1 4342 mov %g2, %l2 4343 cmp %i3, %i2 4344 mov %g3, %l3 4345 mov %g4, %l4 4346 mov %g5, %l5 4347 movle %icc, %i2, %i3 4348 mov %g6, %l6 4349 mov %g7, %l7 4350 wrpr %i3, %g0, %pil 4351 jmpl %i4, %o7 4352 ! Enable 64-bit addresses for the prom 4353#if defined(_LP64) 4354 wrpr %g0, PSTATE_PROM, %pstate 4355#else 4356 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 4357#endif 4358 wrpr %l0, 0, %pstate 4359 wrpr %i2, 0, %pil 4360 mov %l1, %g1 4361 mov %l2, %g2 4362 mov %l3, %g3 4363 mov %l4, %g4 4364 mov %l5, %g5 4365 mov %l6, %g6 4366 mov %l7, %g7 4367 ret 4368 restore %o0, %g0, %o0 4369 4370/* 4371 * void ofw_exit(cell_t args[]) 4372 */ 4373ENTRY(openfirmware_exit) 4374 STACKFRAME(-CC64FSZ) 4375 flushw ! Flush register windows 4376 4377 wrpr %g0, PIL_HIGH, %pil ! Disable interrupts 4378 sethi %hi(romtba), %l5 4379 LDPTR [%l5 + %lo(romtba)], %l5 4380 wrpr %l5, 0, %tba ! restore the ofw trap table 4381 4382 /* Arrange locked kernel stack as PROM stack */ 4383 set EINTSTACK - CC64FSZ, %l5 4384 4385 andn %l5, 0x0f, %l5 ! Needs to be 16-byte aligned 4386 sub %l5, BIAS, %l5 ! and biased 4387 mov %l5, %sp 4388 flushw 4389 4390 sethi %hi(romp), %l6 4391 LDPTR [%l6 + %lo(romp)], %l6 4392 4393 mov CTX_PRIMARY, %l3 ! set context 0 4394 stxa %g0, [%l3] ASI_DMMU 4395 membar #Sync 4396 4397 wrpr %g0, PSTATE_PROM, %pstate ! Disable interrupts 4398 ! and enable 64-bit addresses 4399 wrpr %g0, 0, %tl ! force trap level 0 4400 call %l6 4401 mov %i0, %o0 4402 NOTREACHED 4403 4404/* 4405 * sp_tlb_flush_pte_us(vaddr_t va, int ctx) 4406 * sp_tlb_flush_pte_usiii(vaddr_t va, int ctx) 4407 * 4408 * Flush tte from both IMMU and DMMU. 4409 * 4410 * This uses %o0-%o5 4411 */ 4412 .align 8 4413ENTRY(sp_tlb_flush_pte_us) 4414#ifdef DEBUG 4415 set pmapdebug, %o3 4416 lduw [%o3], %o3 4417! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 4418 btst 0x0020, %o3 4419 bz,pt %icc, 2f 4420 nop 4421 save %sp, -CC64FSZ, %sp 4422 set 1f, %o0 4423 mov %i1, %o1 4424 andn %i0, 0xfff, %o3 4425 or %o3, 0x010, %o3 4426 call _C_LABEL(printf) 4427 mov %i0, %o2 4428 restore 4429 .data 44301: 4431 .asciz "sp_tlb_flush_pte_us: demap ctx=%x va=%08x res=%x\r\n" 4432 _ALIGN 4433 .text 44342: 4435#endif 4436#ifdef MULTIPROCESSOR 4437 rdpr %pstate, %o3 4438 andn %o3, PSTATE_IE, %o4 ! disable interrupts 4439 wrpr %o4, 0, %pstate 4440#endif 4441 srlx %o0, PG_SHIFT4U, %o0 ! drop unused va bits 4442 mov CTX_SECONDARY, %o2 4443 sllx %o0, PG_SHIFT4U, %o0 4444 ldxa [%o2] ASI_DMMU, %o5 ! Save secondary context 4445 sethi %hi(KERNBASE), %o4 4446 membar #LoadStore 4447 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 4448 membar #Sync 4449 or %o0, DEMAP_PAGE_SECONDARY, %o0 ! Demap page from secondary context only 4450 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 4451 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 4452#ifdef TLB_FLUSH_LOWVA 4453 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 4454 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 4455 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 4456#endif 4457 flush %o4 4458 stxa %o5, [%o2] ASI_DMMU ! Restore secondary context 4459 membar #Sync 4460 retl 4461#ifdef MULTIPROCESSOR 4462 wrpr %o3, %pstate ! restore interrupts 4463#else 4464 nop 4465#endif 4466 4467ENTRY(sp_tlb_flush_pte_usiii) 4468#ifdef DEBUG 4469 set pmapdebug, %o3 4470 lduw [%o3], %o3 4471! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 4472 btst 0x0020, %o3 4473 bz,pt %icc, 2f 4474 nop 4475 save %sp, -CC64FSZ, %sp 4476 set 1f, %o0 4477 mov %i1, %o1 4478 andn %i0, 0xfff, %o3 4479 or %o3, 0x010, %o3 4480 call _C_LABEL(printf) 4481 mov %i0, %o2 4482 restore 4483 .data 44841: 4485 .asciz "sp_tlb_flush_pte_usiii: demap ctx=%x va=%08x res=%x\r\n" 4486 _ALIGN 4487 .text 44882: 4489#endif 4490 ! %o0 = VA [in] 4491 ! %o1 = ctx value [in] / KERNBASE 4492 ! %o2 = CTX_PRIMARY 4493 ! %o3 = saved %tl 4494 ! %o4 = saved %pstate 4495 ! %o5 = saved primary ctx 4496 4497 ! Need this for UP as well 4498 rdpr %pstate, %o4 4499 andn %o4, PSTATE_IE, %o3 ! disable interrupts 4500 wrpr %o3, 0, %pstate 4501 4502 !! 4503 !! Cheetahs do not support flushing the IMMU from secondary context 4504 !! 4505 rdpr %tl, %o3 4506 mov CTX_PRIMARY, %o2 4507 brnz,pt %o3, 1f 4508 andn %o0, 0xfff, %o0 ! drop unused va bits 4509 wrpr %g0, 1, %tl ! Make sure we're NUCLEUS 45101: 4511 ldxa [%o2] ASI_DMMU, %o5 ! Save primary context 4512 membar #LoadStore 4513 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 4514 sethi %hi(KERNBASE), %o1 4515 membar #Sync 4516 or %o0, DEMAP_PAGE_PRIMARY, %o0 4517 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 4518 membar #Sync 4519 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 4520 membar #Sync 4521#ifdef TLB_FLUSH_LOWVA 4522 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 4523 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 4524 membar #Sync 4525 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 4526 membar #Sync 4527#endif 4528 flush %o1 4529 stxa %o5, [%o2] ASI_DMMU ! Restore primary context 4530 membar #Sync 4531 brnz,pt %o3, 1f 4532 flush %o1 4533 wrpr %g0, %o3, %tl ! Return to kernel mode. 45341: 4535 retl 4536 wrpr %o4, %pstate ! restore interrupts 4537 4538 4539/* 4540 * sp_tlb_flush_all_us(void) 4541 * sp_tlb_flush_all_usiii(void) 4542 * 4543 * Flush all user TLB entries from both IMMU and DMMU. 4544 * We have both UltraSPARC I+II, and UltraSPARC >=III versions. 4545 */ 4546 .align 8 4547ENTRY(sp_tlb_flush_all_us) 4548 rdpr %pstate, %o3 4549 andn %o3, PSTATE_IE, %o4 ! disable interrupts 4550 wrpr %o4, 0, %pstate 4551 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 4552 set CTX_SECONDARY, %o4 4553 ldxa [%o4] ASI_DMMU, %o4 ! save secondary context 4554 set CTX_MASK, %o5 4555 membar #Sync 4556 4557 ! %o0 = loop counter 4558 ! %o1 = ctx value 4559 ! %o2 = TLB tag value 4560 ! %o3 = saved %pstate 4561 ! %o4 = saved primary ctx 4562 ! %o5 = CTX_MASK 4563 ! %xx = saved %tl 4564 45650: 4566 ldxa [%o0] ASI_DMMU_TLB_TAG, %o2 ! fetch the TLB tag 4567 andcc %o2, %o5, %o1 ! context 0? 4568 bz,pt %xcc, 1f ! if so, skip 4569 mov CTX_SECONDARY, %o2 4570 4571 stxa %o1, [%o2] ASI_DMMU ! set the context 4572 set DEMAP_CTX_SECONDARY, %o2 4573 membar #Sync 4574 stxa %o2, [%o2] ASI_DMMU_DEMAP ! do the demap 4575 membar #Sync 4576 45771: 4578 dec 8, %o0 4579 brgz,pt %o0, 0b ! loop over all entries 4580 nop 4581 4582/* 4583 * now do the IMMU 4584 */ 4585 4586 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 4587 45880: 4589 ldxa [%o0] ASI_IMMU_TLB_TAG, %o2 ! fetch the TLB tag 4590 andcc %o2, %o5, %o1 ! context 0? 4591 bz,pt %xcc, 1f ! if so, skip 4592 mov CTX_SECONDARY, %o2 4593 4594 stxa %o1, [%o2] ASI_DMMU ! set the context 4595 set DEMAP_CTX_SECONDARY, %o2 4596 membar #Sync 4597 stxa %o2, [%o2] ASI_IMMU_DEMAP ! do the demap 4598 membar #Sync 4599 46001: 4601 dec 8, %o0 4602 brgz,pt %o0, 0b ! loop over all entries 4603 nop 4604 4605 set CTX_SECONDARY, %o2 4606 stxa %o4, [%o2] ASI_DMMU ! restore secondary ctx 4607 sethi %hi(KERNBASE), %o4 4608 membar #Sync 4609 flush %o4 4610 retl 4611 wrpr %o3, %pstate 4612 4613 .align 8 4614ENTRY(sp_tlb_flush_all_usiii) 4615 rdpr %tl, %o5 4616 brnz,pt %o5, 1f 4617 set DEMAP_ALL, %o2 4618 wrpr 1, %tl 46191: 4620 rdpr %pstate, %o3 4621 andn %o3, PSTATE_IE, %o4 ! disable interrupts 4622 wrpr %o4, 0, %pstate 4623 4624 stxa %o2, [%o2] ASI_IMMU_DEMAP 4625 membar #Sync 4626 stxa %o2, [%o2] ASI_DMMU_DEMAP 4627 4628 sethi %hi(KERNBASE), %o4 4629 membar #Sync 4630 flush %o4 4631 4632 wrpr %o5, %tl 4633 retl 4634 wrpr %o3, %pstate 4635 4636/* 4637 * sp_blast_dcache(int dcache_size, int dcache_line_size) 4638 * 4639 * Clear out all of D$ regardless of contents 4640 */ 4641 .align 8 4642ENTRY(sp_blast_dcache) 4643/* 4644 * We turn off interrupts for the duration to prevent RED exceptions. 4645 */ 4646#ifdef PROF 4647 save %sp, -CC64FSZ, %sp 4648#endif 4649 4650 rdpr %pstate, %o3 4651 sub %o0, %o1, %o0 4652 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 4653 wrpr %o4, 0, %pstate 46541: 4655 stxa %g0, [%o0] ASI_DCACHE_TAG 4656 membar #Sync 4657 brnz,pt %o0, 1b 4658 sub %o0, %o1, %o0 4659 4660 sethi %hi(KERNBASE), %o2 4661 flush %o2 4662 membar #Sync 4663#ifdef PROF 4664 wrpr %o3, %pstate 4665 ret 4666 restore 4667#else 4668 retl 4669 wrpr %o3, %pstate 4670#endif 4671 4672#ifdef MULTIPROCESSOR 4673/* 4674 * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size) 4675 * 4676 * Clear out all of D$ regardless of contents 4677 * 4678 * On entry: 4679 * %g2 = dcache_size 4680 * %g3 = dcache_line_size 4681 */ 4682 .align 8 4683ENTRY(sparc64_ipi_blast_dcache) 4684 sub %g2, %g3, %g2 46851: 4686 stxa %g0, [%g2] ASI_DCACHE_TAG 4687 membar #Sync 4688 brnz,pt %g2, 1b 4689 sub %g2, %g3, %g2 4690 4691 sethi %hi(KERNBASE), %g5 4692 flush %g5 4693 membar #Sync 4694 4695 ba,a ret_from_intr_vector 4696 nop 4697#endif /* MULTIPROCESSOR */ 4698 4699/* 4700 * blast_icache_us() 4701 * blast_icache_usiii() 4702 * 4703 * Clear out all of I$ regardless of contents 4704 * Does not modify %o0 4705 * 4706 * We turn off interrupts for the duration to prevent RED exceptions. 4707 * For the Cheetah version, we also have to to turn off the I$ during this as 4708 * ASI_ICACHE_TAG accesses interfere with coherency. 4709 */ 4710 .align 8 4711ENTRY(blast_icache_us) 4712 rdpr %pstate, %o3 4713 sethi %hi(icache_size), %o1 4714 ld [%o1 + %lo(icache_size)], %o1 4715 sethi %hi(icache_line_size), %o2 4716 ld [%o2 + %lo(icache_line_size)], %o2 4717 sub %o1, %o2, %o1 4718 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 4719 wrpr %o4, 0, %pstate 47201: 4721 stxa %g0, [%o1] ASI_ICACHE_TAG 4722 brnz,pt %o1, 1b 4723 sub %o1, %o2, %o1 4724 sethi %hi(KERNBASE), %o5 4725 flush %o5 4726 membar #Sync 4727 retl 4728 wrpr %o3, %pstate 4729 4730 .align 8 4731ENTRY(blast_icache_usiii) 4732 rdpr %pstate, %o3 4733 sethi %hi(icache_size), %o1 4734 ld [%o1 + %lo(icache_size)], %o1 4735 sethi %hi(icache_line_size), %o2 4736 ld [%o2 + %lo(icache_line_size)], %o2 4737 sub %o1, %o2, %o1 4738 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 4739 wrpr %o4, 0, %pstate 4740 ldxa [%g0] ASI_MCCR, %o5 4741 andn %o5, MCCR_ICACHE_EN, %o4 ! Turn off the I$ 4742 stxa %o4, [%g0] ASI_MCCR 4743 flush %g0 47441: 4745 stxa %g0, [%o1] ASI_ICACHE_TAG 4746 membar #Sync 4747 brnz,pt %o1, 1b 4748 sub %o1, %o2, %o1 4749 stxa %o5, [%g0] ASI_MCCR ! Restore the I$ 4750 flush %g0 4751 retl 4752 wrpr %o3, %pstate 4753 4754/* 4755 * dcache_flush_page_us(paddr_t pa) 4756 * dcache_flush_page_usiii(paddr_t pa) 4757 * 4758 * Clear one page from D$. 4759 * 4760 */ 4761 .align 8 4762ENTRY(dcache_flush_page_us) 4763#ifndef _LP64 4764 COMBINE(%o0, %o1, %o0) 4765#endif 4766 mov -1, %o1 ! Generate mask for tag: bits [29..2] 4767 srlx %o0, 13-2, %o2 ! Tag is PA bits <40:13> in bits <29:2> 4768 clr %o4 4769 srl %o1, 2, %o1 ! Now we have bits <29:0> set 4770 set (2*NBPG), %o5 4771 ba,pt %icc, 1f 4772 andn %o1, 3, %o1 ! Now we have bits <29:2> set 4773 4774 .align 8 47751: 4776 ldxa [%o4] ASI_DCACHE_TAG, %o3 4777 mov %o4, %o0 4778 deccc 32, %o5 4779 bl,pn %icc, 2f 4780 inc 32, %o4 4781 4782 xor %o3, %o2, %o3 4783 andcc %o3, %o1, %g0 4784 bne,pt %xcc, 1b 4785 membar #LoadStore 4786 4787 stxa %g0, [%o0] ASI_DCACHE_TAG 4788 ba,pt %icc, 1b 4789 membar #StoreLoad 47902: 4791 4792 sethi %hi(KERNBASE), %o5 4793 flush %o5 4794 retl 4795 membar #Sync 4796 4797 .align 8 4798ENTRY(dcache_flush_page_usiii) 4799#ifndef _LP64 4800 COMBINE(%o0, %o1, %o0) 4801#endif 4802 set NBPG, %o1 4803 sethi %hi(dcache_line_size), %o2 4804 add %o0, %o1, %o1 ! end address 4805 ld [%o2 + %lo(dcache_line_size)], %o2 4806 48071: 4808 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 4809 add %o0, %o2, %o0 4810 cmp %o0, %o1 4811 bl,pt %xcc, 1b 4812 nop 4813 4814 sethi %hi(KERNBASE), %o5 4815 flush %o5 4816 retl 4817 membar #Sync 4818 4819/* 4820 * cache_flush_phys_us(paddr_t, psize_t, int); 4821 * cache_flush_phys_usiii(paddr_t, psize_t, int); 4822 * 4823 * Clear a set of paddrs from the D$, I$ and if param3 is 4824 * non-zero, E$. (E$ is not supported yet). 4825 */ 4826 4827 .align 8 4828ENTRY(cache_flush_phys_us) 4829#ifndef _LP64 4830 COMBINE(%o0, %o1, %o0) 4831 COMBINE(%o2, %o3, %o1) 4832 mov %o4, %o2 4833#endif 4834#ifdef DEBUG 4835 tst %o2 ! Want to clear E$? 4836 tnz 1 ! Error! 4837#endif 4838 add %o0, %o1, %o1 ! End PA 4839 dec %o1 4840 4841 !! 4842 !! Both D$ and I$ tags match pa bits 42-13, but 4843 !! they are shifted different amounts. So we'll 4844 !! generate a mask for bits 40-13. 4845 !! 4846 4847 mov -1, %o2 ! Generate mask for tag: bits [40..13] 4848 srl %o2, 5, %o2 ! 32-5 = [27..0] 4849 sllx %o2, 13, %o2 ! 27+13 = [40..13] 4850 4851 and %o2, %o0, %o0 ! Mask away uninteresting bits 4852 and %o2, %o1, %o1 ! (probably not necessary) 4853 4854 set (2*NBPG), %o5 4855 clr %o4 48561: 4857 ldxa [%o4] ASI_DCACHE_TAG, %o3 4858 sllx %o3, 40-29, %o3 ! Shift D$ tag into place 4859 and %o3, %o2, %o3 ! Mask out trash 4860 4861 cmp %o0, %o3 4862 blt,pt %xcc, 2f ! Too low 4863 cmp %o1, %o3 4864 bgt,pt %xcc, 2f ! Too high 4865 nop 4866 4867 membar #LoadStore 4868 stxa %g0, [%o4] ASI_DCACHE_TAG ! Just right 4869 membar #Sync 48702: 4871 ldda [%o4] ASI_ICACHE_TAG, %g0 ! Tag goes in %g1 4872 sllx %g1, 40-35, %g1 ! Shift I$ tag into place 4873 and %g1, %o2, %g1 ! Mask out trash 4874 cmp %o0, %g1 4875 blt,pt %xcc, 3f 4876 cmp %o1, %g1 4877 bgt,pt %xcc, 3f 4878 nop 4879 stxa %g0, [%o4] ASI_ICACHE_TAG 48803: 4881 membar #StoreLoad 4882 dec 32, %o5 4883 brgz,pt %o5, 1b 4884 inc 32, %o4 4885 4886 sethi %hi(KERNBASE), %o5 4887 flush %o5 4888 retl 4889 membar #Sync 4890 4891 .align 8 4892ENTRY(cache_flush_phys_usiii) 4893#ifndef _LP64 4894 COMBINE(%o0, %o1, %o0) 4895 COMBINE(%o2, %o3, %o1) 4896 mov %o4, %o2 4897#endif 4898#ifdef DEBUG 4899 tst %o2 ! Want to clear E$? 4900 tnz 1 ! Error! 4901#endif 4902 add %o0, %o1, %o1 ! End PA 4903 sethi %hi(dcache_line_size), %o3 4904 ld [%o3 + %lo(dcache_line_size)], %o3 4905 sethi %hi(KERNBASE), %o5 49061: 4907 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 4908 add %o0, %o3, %o0 4909 cmp %o0, %o1 4910 bl,pt %xcc, 1b 4911 nop 4912 4913 /* don't need to flush the I$ on cheetah */ 4914 4915 flush %o5 4916 retl 4917 membar #Sync 4918 4919#ifdef COMPAT_16 4920#ifdef _LP64 4921/* 4922 * XXXXX Still needs lotsa cleanup after sendsig is complete and offsets are known 4923 * 4924 * The following code is copied to the top of the user stack when each 4925 * process is exec'ed, and signals are `trampolined' off it. 4926 * 4927 * When this code is run, the stack looks like: 4928 * [%sp] 128 bytes to which registers can be dumped 4929 * [%sp + 128] signal number (goes in %o0) 4930 * [%sp + 128 + 4] signal code (goes in %o1) 4931 * [%sp + 128 + 8] first word of saved state (sigcontext) 4932 * . 4933 * . 4934 * . 4935 * [%sp + NNN] last word of saved state 4936 * (followed by previous stack contents or top of signal stack). 4937 * The address of the function to call is in %g1; the old %g1 and %o0 4938 * have already been saved in the sigcontext. We are running in a clean 4939 * window, all previous windows now being saved to the stack. 4940 * 4941 * Note that [%sp + 128 + 8] == %sp + 128 + 16. The copy at %sp+128+8 4942 * will eventually be removed, with a hole left in its place, if things 4943 * work out. 4944 */ 4945ENTRY_NOPROFILE(sigcode) 4946 /* 4947 * XXX the `save' and `restore' below are unnecessary: should 4948 * replace with simple arithmetic on %sp 4949 * 4950 * Make room on the stack for 64 %f registers + %fsr. This comes 4951 * out to 64*4+8 or 264 bytes, but this must be aligned to a multiple 4952 * of 64, or 320 bytes. 4953 */ 4954 save %sp, -CC64FSZ - 320, %sp 4955 mov %g2, %l2 ! save globals in %l registers 4956 mov %g3, %l3 4957 mov %g4, %l4 4958 mov %g5, %l5 4959 mov %g6, %l6 4960 mov %g7, %l7 4961 /* 4962 * Saving the fpu registers is expensive, so do it iff it is 4963 * enabled and dirty. 4964 */ 4965 rd %fprs, %l0 4966 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 4967 bz,pt %icc, 2f 4968 btst FPRS_DL, %l0 ! test dl 4969 bz,pt %icc, 1f 4970 btst FPRS_DU, %l0 ! test du 4971 4972 ! fpu is enabled, oh well 4973 stx %fsr, [%sp + CC64FSZ + BIAS + 0] 4974 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 4975 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 4976 stda %f0, [%l0] ASI_BLK_P 4977 inc BLOCK_SIZE, %l0 4978 stda %f16, [%l0] ASI_BLK_P 49791: 4980 bz,pt %icc, 2f 4981 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 4982 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 4983 add %l0, 2*BLOCK_SIZE, %l0 ! and skip what we already stored 4984 stda %f32, [%l0] ASI_BLK_P 4985 inc BLOCK_SIZE, %l0 4986 stda %f48, [%l0] ASI_BLK_P 49872: 4988 membar #Sync 4989 rd %fprs, %l0 ! reload fprs copy, for checking after 4990 rd %y, %l1 ! in any case, save %y 4991 lduw [%fp + BIAS + 128], %o0 ! sig 4992 lduw [%fp + BIAS + 128 + 4], %o1 ! code 4993 call %g1 ! (*sa->sa_handler)(sig,code,scp) 4994 add %fp, BIAS + 128 + 8, %o2 ! scp 4995 wr %l1, %g0, %y ! in any case, restore %y 4996 4997 /* 4998 * Now that the handler has returned, re-establish all the state 4999 * we just saved above, then do a sigreturn. 5000 */ 5001 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 5002 bz,pt %icc, 2f 5003 btst FPRS_DL, %l0 ! test dl 5004 bz,pt %icc, 1f 5005 btst FPRS_DU, %l0 ! test du 5006 5007 ldx [%sp + CC64FSZ + BIAS + 0], %fsr 5008 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 5009 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 5010 ldda [%l0] ASI_BLK_P, %f0 5011 inc BLOCK_SIZE, %l0 5012 ldda [%l0] ASI_BLK_P, %f16 50131: 5014 bz,pt %icc, 2f 5015 nop 5016 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 5017 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 5018 inc 2*BLOCK_SIZE, %l0 ! and skip what we already loaded 5019 ldda [%l0] ASI_BLK_P, %f32 5020 inc BLOCK_SIZE, %l0 5021 ldda [%l0] ASI_BLK_P, %f48 50222: 5023 mov %l2, %g2 5024 mov %l3, %g3 5025 mov %l4, %g4 5026 mov %l5, %g5 5027 mov %l6, %g6 5028 mov %l7, %g7 5029 membar #Sync 5030 5031 restore %g0, SYS_compat_16___sigreturn14, %g1 ! get registers back & set syscall # 5032 add %sp, BIAS + 128 + 8, %o0! compute scp 5033! andn %o0, 0x0f, %o0 5034 t ST_SYSCALL ! sigreturn(scp) 5035 ! sigreturn does not return unless it fails 5036 mov SYS_exit, %g1 ! exit(errno) 5037 t ST_SYSCALL 5038 /* NOTREACHED */ 5039 5040 .globl _C_LABEL(esigcode) 5041_C_LABEL(esigcode): 5042#endif 5043 5044#if !defined(_LP64) 5045 5046#define SIGCODE_NAME sigcode 5047#define ESIGCODE_NAME esigcode 5048#define SIGRETURN_NAME SYS_compat_16___sigreturn14 5049#define EXIT_NAME SYS_exit 5050 5051#include "sigcode32.s" 5052 5053#endif 5054#endif 5055 5056/* 5057 * getfp() - get stack frame pointer 5058 */ 5059ENTRY(getfp) 5060 retl 5061 mov %fp, %o0 5062 5063/* 5064 * nothing MD to do in the idle loop 5065 */ 5066ENTRY(cpu_idle) 5067 retl 5068 nop 5069 5070/* 5071 * cpu_switchto() switches to an lwp to run and runs it, saving the 5072 * current one away. 5073 * 5074 * stuct lwp * cpu_switchto(struct lwp *current, struct lwp *next) 5075 * Switch to the specified next LWP 5076 * Arguments: 5077 * i0 'struct lwp *' of the current LWP 5078 * i1 'struct lwp *' of the LWP to switch to 5079 * Returns: 5080 * the old lwp switched away from 5081 */ 5082ENTRY(cpu_switchto) 5083 save %sp, -CC64FSZ, %sp 5084 /* 5085 * REGISTER USAGE AT THIS POINT: 5086 * %l1 = newpcb 5087 * %l3 = new trapframe 5088 * %l4 = new l->l_proc 5089 * %l5 = pcb of oldlwp 5090 * %l6 = %hi(CPCB) 5091 * %l7 = %hi(CURLWP) 5092 * %i0 = oldlwp 5093 * %i1 = lwp 5094 * %o0 = tmp 1 5095 * %o1 = tmp 2 5096 * %o2 = tmp 3 5097 * %o3 = tmp 4 5098 */ 5099 5100 flushw ! save all register windows except this one 5101 wrpr %g0, PSTATE_KERN, %pstate ! make sure we're on normal globals 5102 ! with traps turned off 5103 5104 brz,pn %i0, 1f 5105 sethi %hi(CPCB), %l6 5106 5107 rdpr %pstate, %o1 ! oldpstate = %pstate; 5108 LDPTR [%i0 + L_PCB], %l5 5109 5110 stx %i7, [%l5 + PCB_PC] 5111 stx %i6, [%l5 + PCB_SP] 5112 sth %o1, [%l5 + PCB_PSTATE] 5113 5114 rdpr %cwp, %o2 ! Useless 5115 stb %o2, [%l5 + PCB_CWP] 5116 51171: 5118 sethi %hi(CURLWP), %l7 5119 5120 LDPTR [%i1 + L_PCB], %l1 ! newpcb = l->l_pcb; 5121 5122 /* 5123 * Load the new lwp. To load, we must change stacks and 5124 * alter cpcb and the window control registers, hence we must 5125 * keep interrupts disabled. 5126 */ 5127 5128 STPTR %i1, [%l7 + %lo(CURLWP)] ! curlwp = l; 5129 STPTR %l1, [%l6 + %lo(CPCB)] ! cpcb = newpcb; 5130 5131 ldx [%l1 + PCB_SP], %i6 5132 ldx [%l1 + PCB_PC], %i7 5133 5134 wrpr %g0, 0, %otherwin ! These two insns should be redundant 5135 wrpr %g0, 0, %canrestore 5136 rdpr %ver, %o3 5137 and %o3, CWP, %o3 5138 wrpr %g0, %o3, %cleanwin 5139 dec 1, %o3 ! NWINDOWS-1-1 5140 wrpr %o3, %cansave 5141 5142 /* finally, enable traps */ 5143 wrpr %g0, PSTATE_INTR, %pstate 5144 5145 !flushw 5146 !membar #Sync 5147 5148 /* 5149 * Check for restartable atomic sequences (RAS) 5150 */ 5151 LDPTR [%i1 + L_PROC], %l4 ! now %l4 points to p 5152 mov %l4, %o0 ! p is first arg to ras_lookup 5153 LDPTR [%o0 + P_RASLIST], %o1 ! any RAS in p? 5154 brz,pt %o1, Lsw_noras ! no, skip RAS check 5155 LDPTR [%i1 + L_TF], %l3 ! pointer to trap frame 5156 call _C_LABEL(ras_lookup) 5157 LDPTR [%l3 + TF_PC], %o1 5158 cmp %o0, -1 5159 be,pt %xcc, Lsw_noras 5160 add %o0, 4, %o1 5161 STPTR %o0, [%l3 + TF_PC] ! store rewound %pc 5162 STPTR %o1, [%l3 + TF_NPC] ! and %npc 5163 5164Lsw_noras: 5165 5166 /* 5167 * We are resuming the process that was running at the 5168 * call to switch(). Just set psr ipl and return. 5169 */ 5170! wrpr %g0, 0, %cleanwin ! DEBUG 5171 clr %g4 ! This needs to point to the base of the data segment 5172 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 5173 !wrpr %g0, PSTATE_INTR, %pstate 5174 ret 5175 restore %i0, %g0, %o0 ! return old curlwp 5176 5177/* 5178 * Snapshot the current process so that stack frames are up to date. 5179 * Only used just before a crash dump. 5180 */ 5181ENTRY(snapshot) 5182 rdpr %pstate, %o1 ! save psr 5183 stx %o7, [%o0 + PCB_PC] ! save pc 5184 stx %o6, [%o0 + PCB_SP] ! save sp 5185 rdpr %pil, %o2 5186 sth %o1, [%o0 + PCB_PSTATE] 5187 rdpr %cwp, %o3 5188 stb %o2, [%o0 + PCB_PIL] 5189 stb %o3, [%o0 + PCB_CWP] 5190 5191 flushw 5192 save %sp, -CC64FSZ, %sp 5193 flushw 5194 ret 5195 restore 5196 5197/* 5198 * cpu_lwp_fork() arranges for lwp_trampoline() to run when the 5199 * nascent lwp is selected by switch(). 5200 * 5201 * The switch frame will contain pointer to struct lwp of this lwp in 5202 * %l2, a pointer to the function to call in %l0, and an argument to 5203 * pass to it in %l1 (we abuse the callee-saved registers). 5204 * 5205 * We enter lwp_trampoline as if we are "returning" from 5206 * cpu_switchto(), so %o0 contains previous lwp (the one we are 5207 * switching from) that we pass to lwp_startup(). 5208 * 5209 * If the function *(%l0) returns, we arrange for an immediate return 5210 * to user mode. This happens in two known cases: after execve(2) of 5211 * init, and when returning a child to user mode after a fork(2). 5212 * 5213 * If were setting up a kernel thread, the function *(%l0) will not 5214 * return. 5215 */ 5216ENTRY(lwp_trampoline) 5217 /* 5218 * Note: cpu_lwp_fork() has set up a stack frame for us to run 5219 * in, so we can call other functions from here without using 5220 * `save ... restore'. 5221 */ 5222 5223 ! newlwp in %l2, oldlwp in %o0 5224 call lwp_startup 5225 mov %l2, %o1 5226 5227 call %l0 ! re-use current frame 5228 mov %l1, %o0 5229 5230 /* 5231 * Going to userland - set proper tstate in trap frame 5232 */ 5233 set (ASI_PRIMARY_NO_FAULT<<TSTATE_ASI_SHIFT)|((PSTATE_USER)<<TSTATE_PSTATE_SHIFT), %g1 5234 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 5235 5236 /* 5237 * Here we finish up as in syscall, but simplified. 5238 */ 5239 ba,a,pt %icc, return_from_trap 5240 nop 5241 5242 /* 5243 * Like lwp_trampoline, but for cpu_setfunc(), i.e. without newlwp 5244 * arguement and will not call lwp_startup. 5245 */ 5246ENTRY(setfunc_trampoline) 5247 call %l0 ! re-use current frame 5248 mov %l1, %o0 5249 ba,a,pt %icc, return_from_trap 5250 nop 5251 5252/* 5253 * pmap_zero_page_phys(pa) 5254 * 5255 * Zero one page physically addressed 5256 * 5257 * Block load/store ASIs do not exist for physical addresses, 5258 * so we won't use them. 5259 * 5260 * We will execute a flush at the end to sync the I$. 5261 * 5262 * This version expects to have the dcache_flush_page_all(pa) 5263 * to have been called before calling into here. 5264 */ 5265ENTRY(pmap_zero_page_phys) 5266#ifndef _LP64 5267 COMBINE(%o0, %o1, %o0) 5268#endif 5269#ifdef DEBUG 5270 set pmapdebug, %o4 5271 ld [%o4], %o4 5272 btst 0x80, %o4 ! PDB_COPY 5273 bz,pt %icc, 3f 5274 nop 5275 save %sp, -CC64FSZ, %sp 5276 set 2f, %o0 5277 call printf 5278 mov %i0, %o1 5279! ta 1; nop 5280 restore 5281 .data 52822: .asciz "pmap_zero_page(%p)\n" 5283 _ALIGN 5284 .text 52853: 5286#endif 5287 set NBPG, %o2 ! Loop count 5288 wr %g0, ASI_PHYS_CACHED, %asi 52891: 5290 /* Unroll the loop 8 times */ 5291 stxa %g0, [%o0 + 0x00] %asi 5292 deccc 0x40, %o2 5293 stxa %g0, [%o0 + 0x08] %asi 5294 stxa %g0, [%o0 + 0x10] %asi 5295 stxa %g0, [%o0 + 0x18] %asi 5296 stxa %g0, [%o0 + 0x20] %asi 5297 stxa %g0, [%o0 + 0x28] %asi 5298 stxa %g0, [%o0 + 0x30] %asi 5299 stxa %g0, [%o0 + 0x38] %asi 5300 bg,pt %icc, 1b 5301 inc 0x40, %o0 5302 5303 sethi %hi(KERNBASE), %o3 5304 flush %o3 5305 retl 5306 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Make C code happy 5307 5308/* 5309 * pmap_copy_page_phys(paddr_t src, paddr_t dst) 5310 * 5311 * Copy one page physically addressed 5312 * We need to use a global reg for ldxa/stxa 5313 * so the top 32-bits cannot be lost if we take 5314 * a trap and need to save our stack frame to a 5315 * 32-bit stack. We will unroll the loop by 4 to 5316 * improve performance. 5317 * 5318 * This version expects to have the dcache_flush_page_all(pa) 5319 * to have been called before calling into here. 5320 * 5321 */ 5322ENTRY(pmap_copy_page_phys) 5323#ifndef _LP64 5324 COMBINE(%o0, %o1, %o0) 5325 COMBINE(%o2, %o3, %o1) 5326#endif 5327#ifdef DEBUG 5328 set pmapdebug, %o4 5329 ld [%o4], %o4 5330 btst 0x80, %o4 ! PDB_COPY 5331 bz,pt %icc, 3f 5332 nop 5333 save %sp, -CC64FSZ, %sp 5334 mov %i0, %o1 5335 set 2f, %o0 5336 call printf 5337 mov %i1, %o2 5338! ta 1; nop 5339 restore 5340 .data 53412: .asciz "pmap_copy_page(%p,%p)\n" 5342 _ALIGN 5343 .text 53443: 5345#endif 5346#if 1 5347 set NBPG, %o2 5348 wr %g0, ASI_PHYS_CACHED, %asi 53491: 5350 ldxa [%o0 + 0x00] %asi, %g1 5351 ldxa [%o0 + 0x08] %asi, %o3 5352 ldxa [%o0 + 0x10] %asi, %o4 5353 ldxa [%o0 + 0x18] %asi, %o5 5354 inc 0x20, %o0 5355 deccc 0x20, %o2 5356 stxa %g1, [%o1 + 0x00] %asi 5357 stxa %o3, [%o1 + 0x08] %asi 5358 stxa %o4, [%o1 + 0x10] %asi 5359 stxa %o5, [%o1 + 0x18] %asi 5360 bg,pt %icc, 1b ! We don't care about pages >4GB 5361 inc 0x20, %o1 5362 retl 5363 wr %g0, ASI_PRIMARY_NOFAULT, %asi 5364#else 5365 set NBPG, %o3 5366 add %o3, %o0, %o3 5367 mov %g1, %o4 ! Save g1 53681: 5369 ldxa [%o0] ASI_PHYS_CACHED, %g1 5370 inc 8, %o0 5371 cmp %o0, %o3 5372 stxa %g1, [%o1] ASI_PHYS_CACHED 5373 bl,pt %icc, 1b ! We don't care about pages >4GB 5374 inc 8, %o1 5375 retl 5376 mov %o4, %g1 ! Restore g1 5377#endif 5378 5379/* 5380 * extern int64_t pseg_get_real(struct pmap *pm, vaddr_t addr); 5381 * 5382 * Return TTE at addr in pmap. Uses physical addressing only. 5383 * pmap->pm_physaddr must by the physical address of pm_segs 5384 * 5385 */ 5386ENTRY(pseg_get_real) 5387! flushw ! Make sure we don't have stack probs & lose hibits of %o 5388 ldx [%o0 + PM_PHYS], %o2 ! pmap->pm_segs 5389 5390 srax %o1, HOLESHIFT, %o3 ! Check for valid address 5391 brz,pt %o3, 0f ! Should be zero or -1 5392 inc %o3 ! Make -1 -> 0 5393 brnz,pn %o3, 1f ! Error! In hole! 53940: 5395 srlx %o1, STSHIFT, %o3 5396 and %o3, STMASK, %o3 ! Index into pm_segs 5397 sll %o3, 3, %o3 5398 add %o2, %o3, %o2 5399 DLFLUSH(%o2,%o3) 5400 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page directory pointer 5401 DLFLUSH2(%o3) 5402 5403 srlx %o1, PDSHIFT, %o3 5404 and %o3, PDMASK, %o3 5405 sll %o3, 3, %o3 5406 brz,pn %o2, 1f ! NULL entry? check somewhere else 5407 add %o2, %o3, %o2 5408 DLFLUSH(%o2,%o3) 5409 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page table pointer 5410 DLFLUSH2(%o3) 5411 5412 srlx %o1, PTSHIFT, %o3 ! Convert to ptab offset 5413 and %o3, PTMASK, %o3 5414 sll %o3, 3, %o3 5415 brz,pn %o2, 1f ! NULL entry? check somewhere else 5416 add %o2, %o3, %o2 5417 DLFLUSH(%o2,%o3) 5418 ldxa [%o2] ASI_PHYS_CACHED, %o0 5419 DLFLUSH2(%o3) 5420 brgez,pn %o0, 1f ! Entry invalid? Punt 5421 btst 1, %sp 5422 bz,pn %icc, 0f ! 64-bit mode? 5423 nop 5424 retl ! Yes, return full value 5425 nop 54260: 5427#if 1 5428 srl %o0, 0, %o1 5429 retl ! No, generate a %o0:%o1 double 5430 srlx %o0, 32, %o0 5431#else 5432 DLFLUSH(%o2,%o3) 5433 ldda [%o2] ASI_PHYS_CACHED, %o0 5434 DLFLUSH2(%o3) 5435 retl ! No, generate a %o0:%o1 double 5436 nop 5437#endif 54381: 5439#ifndef _LP64 5440 clr %o1 5441#endif 5442 retl 5443 clr %o0 5444 5445/* 5446 * In 32-bit mode: 5447 * 5448 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 5449 * int64_t tte %o2:%o3, paddr_t spare %o4:%o5); 5450 * 5451 * In 64-bit mode: 5452 * 5453 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 5454 * int64_t tte %o2, paddr_t spare %o3); 5455 * 5456 * Set a pseg entry to a particular TTE value. Return values are: 5457 * 5458 * -2 addr in hole 5459 * 0 success (spare was not used if given) 5460 * 1 failure (spare was not given, but one is needed) 5461 * 2 success (spare was given, used for L2) 5462 * 3 failure (spare was given, used for L2, another is needed for L3) 5463 * 4 success (spare was given, used for L3) 5464 * 5465 * rv == 0 success, spare not used if one was given 5466 * rv & 4 spare was used for L3 5467 * rv & 2 spare was used for L2 5468 * rv & 1 failure, spare is needed 5469 * 5470 * (NB: nobody in pmap checks for the virtual hole, so the system will hang.) 5471 * The way to call this is: first just call it without a spare page. 5472 * If that fails, allocate a page and try again, passing the paddr of the 5473 * new page as the spare. 5474 * If spare is non-zero it is assumed to be the address of a zeroed physical 5475 * page that can be used to generate a directory table or page table if needed. 5476 * 5477 * We keep track of valid (A_TLB_V bit set) and wired (A_TLB_TSB_LOCK bit set) 5478 * mappings that are set here. We check both bits on the new data entered 5479 * and increment counts, as well as decrementing counts if the bits are set 5480 * in the value replaced by this call. 5481 * The counters are 32 bit or 64 bit wide, depending on the kernel type we are 5482 * running! 5483 */ 5484ENTRY(pseg_set_real) 5485#ifndef _LP64 5486 clruw %o1 ! Zero extend 5487 COMBINE(%o2, %o3, %o2) 5488 COMBINE(%o4, %o5, %o3) 5489#endif 5490 !! 5491 !! However we managed to get here we now have: 5492 !! 5493 !! %o0 = *pmap 5494 !! %o1 = addr 5495 !! %o2 = tte 5496 !! %o3 = paddr of spare page 5497 !! 5498 srax %o1, HOLESHIFT, %o4 ! Check for valid address 5499 brz,pt %o4, 0f ! Should be zero or -1 5500 inc %o4 ! Make -1 -> 0 5501 brz,pt %o4, 0f 5502 nop 5503#ifdef DEBUG 5504 ta 1 ! Break into debugger 5505#endif 5506 retl 5507 mov -2, %o0 ! Error -- in hole! 5508 55090: 5510 ldx [%o0 + PM_PHYS], %o4 ! pmap->pm_segs 5511 clr %g1 5512 srlx %o1, STSHIFT, %o5 5513 and %o5, STMASK, %o5 5514 sll %o5, 3, %o5 5515 add %o4, %o5, %o4 55160: 5517 DLFLUSH(%o4,%g5) 5518 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load page directory pointer 5519 DLFLUSH2(%g5) 5520 5521 brnz,a,pt %o5, 0f ! Null pointer? 5522 mov %o5, %o4 5523 brz,pn %o3, 9f ! Have a spare? 5524 mov %o3, %o5 5525 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 5526 brnz,pn %o5, 0b ! Something changed? 5527 DLFLUSH(%o4, %o5) 5528 mov %o3, %o4 5529 mov 2, %g1 ! record spare used for L2 5530 clr %o3 ! and not available for L3 55310: 5532 srlx %o1, PDSHIFT, %o5 5533 and %o5, PDMASK, %o5 5534 sll %o5, 3, %o5 5535 add %o4, %o5, %o4 55360: 5537 DLFLUSH(%o4,%g5) 5538 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load table directory pointer 5539 DLFLUSH2(%g5) 5540 5541 brnz,a,pt %o5, 0f ! Null pointer? 5542 mov %o5, %o4 5543 brz,pn %o3, 9f ! Have a spare? 5544 mov %o3, %o5 5545 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 5546 brnz,pn %o5, 0b ! Something changed? 5547 DLFLUSH(%o4, %o4) 5548 mov %o3, %o4 5549 mov 4, %g1 ! record spare used for L3 55500: 5551 srlx %o1, PTSHIFT, %o5 ! Convert to ptab offset 5552 and %o5, PTMASK, %o5 5553 sll %o5, 3, %o5 5554 add %o5, %o4, %o4 5555 5556 DLFLUSH(%o4,%g5) 5557 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! save old value in %o5 5558 stxa %o2, [%o4] ASI_PHYS_CACHED ! Easier than shift+or 5559 DLFLUSH2(%g5) 5560 5561 !! at this point we have: 5562 !! %g1 = return value 5563 !! %o0 = struct pmap * (where the counts are) 5564 !! %o2 = new TTE 5565 !! %o5 = old TTE 5566 5567 !! see if stats needs an update 5568 set A_TLB_TSB_LOCK, %g5 5569 xor %o2, %o5, %o3 ! %o3 - what changed 5570 5571 brgez,pn %o3, 5f ! has resident changed? (we predict it has) 5572 btst %g5, %o3 ! has wired changed? 5573 5574 LDPTR [%o0 + PM_RESIDENT], %o1 ! gonna update resident count 5575 brlz %o2, 0f 5576 mov 1, %o4 5577 neg %o4 ! new is not resident -> decrement 55780: add %o1, %o4, %o1 5579 STPTR %o1, [%o0 + PM_RESIDENT] 5580 btst %g5, %o3 ! has wired changed? 55815: bz,pt %xcc, 8f ! we predict it's not 5582 btst %g5, %o2 ! don't waste delay slot, check if new one is wired 5583 LDPTR [%o0 + PM_WIRED], %o1 ! gonna update wired count 5584 bnz,pt %xcc, 0f ! if wired changes, we predict it increments 5585 mov 1, %o4 5586 neg %o4 ! new is not wired -> decrement 55870: add %o1, %o4, %o1 5588 STPTR %o1, [%o0 + PM_WIRED] 55898: retl 5590 mov %g1, %o0 ! return %g1 5591 55929: retl 5593 or %g1, 1, %o0 ! spare needed, return flags + 1 5594 5595 5596/* 5597 * clearfpstate() 5598 * 5599 * Drops the current fpu state, without saving it. 5600 */ 5601ENTRY(clearfpstate) 5602 rdpr %pstate, %o1 ! enable FPU 5603 wr %g0, FPRS_FEF, %fprs 5604 or %o1, PSTATE_PEF, %o1 5605 retl 5606 wrpr %o1, 0, %pstate 5607 5608/* 5609 * savefpstate(f) struct fpstate *f; 5610 * 5611 * Store the current FPU state. 5612 * 5613 * Since the kernel may need to use the FPU and we have problems atomically 5614 * testing and enabling the FPU, we leave here with the FPRS_FEF bit set. 5615 * Normally this should be turned on in loadfpstate(). 5616 */ 5617 /* XXXXXXXXXX Assume caller created a proper stack frame */ 5618ENTRY(savefpstate) 5619! flushw ! Make sure we don't have stack probs & lose hibits of %o 5620 rdpr %pstate, %o1 ! enable FP before we begin 5621 rd %fprs, %o5 5622 wr %g0, FPRS_FEF, %fprs 5623 or %o1, PSTATE_PEF, %o1 5624 wrpr %o1, 0, %pstate 5625 5626 stx %fsr, [%o0 + FS_FSR] ! f->fs_fsr = getfsr(); 5627 rd %gsr, %o4 ! Save %gsr 5628 st %o4, [%o0 + FS_GSR] 5629 5630 add %o0, FS_REGS, %o2 5631#ifdef DIAGNOSTIC 5632 btst BLOCK_ALIGN, %o2 ! Needs to be re-executed 5633 bnz,pn %icc, 6f ! Check alignment 5634#endif 5635 st %g0, [%o0 + FS_QSIZE] ! f->fs_qsize = 0; 5636 btst FPRS_DL|FPRS_DU, %o5 ! Both FPU halves clean? 5637 bz,pt %icc, 5f ! Then skip it 5638 5639 btst FPRS_DL, %o5 ! Lower FPU clean? 5640 membar #Sync 5641 bz,a,pt %icc, 1f ! Then skip it, but upper FPU not clean 5642 add %o2, 2*BLOCK_SIZE, %o2 ! Skip a block 5643 5644 stda %f0, [%o2] ASI_BLK_P ! f->fs_f0 = etc; 5645 inc BLOCK_SIZE, %o2 5646 stda %f16, [%o2] ASI_BLK_P 5647 5648 btst FPRS_DU, %o5 ! Upper FPU clean? 5649 bz,pt %icc, 2f ! Then skip it 5650 inc BLOCK_SIZE, %o2 56511: 5652 stda %f32, [%o2] ASI_BLK_P 5653 inc BLOCK_SIZE, %o2 5654 stda %f48, [%o2] ASI_BLK_P 56552: 5656 membar #Sync ! Finish operation so we can 56575: 5658 retl 5659 wr %g0, FPRS_FEF, %fprs ! Mark FPU clean 5660 5661#ifdef DIAGNOSTIC 5662 !! 5663 !! Damn thing is *NOT* aligned on a 64-byte boundary 5664 !! 56656: 5666 wr %g0, FPRS_FEF, %fprs 5667 ! XXX -- we should panic instead of silently entering debugger 5668 ta 1 5669 retl 5670 nop 5671#endif 5672 5673/* 5674 * Load FPU state. 5675 */ 5676 /* XXXXXXXXXX Should test to see if we only need to do a partial restore */ 5677ENTRY(loadfpstate) 5678 flushw ! Make sure we don't have stack probs & lose hibits of %o 5679 rdpr %pstate, %o1 ! enable FP before we begin 5680 ld [%o0 + FS_GSR], %o4 ! Restore %gsr 5681 set PSTATE_PEF, %o2 5682 wr %g0, FPRS_FEF, %fprs 5683 or %o1, %o2, %o1 5684 wrpr %o1, 0, %pstate 5685 ldx [%o0 + FS_FSR], %fsr ! setfsr(f->fs_fsr); 5686 add %o0, FS_REGS, %o3 ! This is zero... 5687#ifdef DIAGNOSTIC 5688 btst BLOCK_ALIGN, %o3 5689 bne,pn %icc, 1f ! Only use block loads on aligned blocks 5690#endif 5691 wr %o4, %g0, %gsr 5692 membar #Sync 5693 ldda [%o3] ASI_BLK_P, %f0 5694 inc BLOCK_SIZE, %o3 5695 ldda [%o3] ASI_BLK_P, %f16 5696 inc BLOCK_SIZE, %o3 5697 ldda [%o3] ASI_BLK_P, %f32 5698 inc BLOCK_SIZE, %o3 5699 ldda [%o3] ASI_BLK_P, %f48 5700 membar #Sync ! Make sure loads are complete 5701 retl 5702 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 5703 5704#ifdef DIAGNOSTIC 5705 !! 5706 !! Damn thing is *NOT* aligned on a 64-byte boundary 5707 !! 57081: 5709 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 5710 ! XXX -- we should panic instead of silently entering debugger 5711 ta 1 5712 retl 5713 nop 5714#endif 5715 5716/* 5717 * ienab_bis(bis) int bis; 5718 * ienab_bic(bic) int bic; 5719 * 5720 * Set and clear bits in the interrupt register. 5721 */ 5722 5723/* 5724 * sun4u has separate asr's for clearing/setting the interrupt mask. 5725 */ 5726ENTRY(ienab_bis) 5727 retl 5728 wr %o0, 0, SET_SOFTINT ! SET_SOFTINT 5729 5730ENTRY(ienab_bic) 5731 retl 5732 wr %o0, 0, CLEAR_SOFTINT ! CLEAR_SOFTINT 5733 5734/* 5735 * send_softint(cpu, level, intrhand) 5736 * 5737 * Send a softint with an intrhand pointer so we can cause a vectored 5738 * interrupt instead of a polled interrupt. This does pretty much the same 5739 * as interrupt_vector. If cpu is -1 then send it to this CPU, if it's -2 5740 * send it to any CPU, otherwise send it to a particular CPU. 5741 * 5742 * XXXX Dispatching to different CPUs is not implemented yet. 5743 */ 5744ENTRY(send_softint) 5745 rdpr %pstate, %g1 5746 andn %g1, PSTATE_IE, %g2 ! clear PSTATE.IE 5747 wrpr %g2, 0, %pstate 5748 5749 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %o3 5750 LDPTR [%o2 + IH_PEND], %o5 5751 or %o3, %lo(CPUINFO_VA+CI_INTRPENDING), %o3 5752 brnz %o5, 1f 5753 sll %o1, PTRSHFT, %o5 ! Find start of table for this IPL 5754 add %o3, %o5, %o3 57552: 5756 LDPTR [%o3], %o5 ! Load list head 5757 STPTR %o5, [%o2+IH_PEND] ! Link our intrhand node in 5758 mov %o2, %o4 5759 CASPTR [%o3] ASI_N, %o5, %o4 5760 cmp %o4, %o5 ! Did it work? 5761 bne,pn CCCR, 2b ! No, try again 5762 .empty 5763 5764 mov 1, %o4 ! Change from level to bitmask 5765 sllx %o4, %o1, %o4 5766 wr %o4, 0, SET_SOFTINT ! SET_SOFTINT 57671: 5768 retl 5769 wrpr %g1, 0, %pstate ! restore PSTATE.IE 5770 5771 5772#define MICROPERSEC (1000000) 5773 5774/* 5775 * delay function 5776 * 5777 * void delay(N) -- delay N microseconds 5778 * 5779 * Register usage: %o0 = "N" number of usecs to go (counts down to zero) 5780 * %o1 = "timerblurb" (stays constant) 5781 * %o2 = counter for 1 usec (counts down from %o1 to zero) 5782 * 5783 * 5784 * ci_cpu_clockrate should be tuned during CPU probe to the CPU 5785 * clockrate in Hz 5786 * 5787 */ 5788ENTRY(delay) ! %o0 = n 5789#if 1 5790 rdpr %tick, %o1 ! Take timer snapshot 5791 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %o2 5792 sethi %hi(MICROPERSEC), %o3 5793 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)], %o4 ! Get scale factor 5794 brnz,pt %o4, 0f 5795 or %o3, %lo(MICROPERSEC), %o3 5796 5797 !! Calculate ticks/usec 5798 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %o4 ! No, we need to calculate it 5799 udivx %o4, %o3, %o4 5800 stx %o4, [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)] ! Save it so we don't need to divide again 58010: 5802 5803 mulx %o0, %o4, %o0 ! Convert usec -> ticks 5804 rdpr %tick, %o2 ! Top of next itr 58051: 5806 sub %o2, %o1, %o3 ! How many ticks have gone by? 5807 sub %o0, %o3, %o4 ! Decrement count by that much 5808 movrgz %o3, %o4, %o0 ! But only if we're decrementing 5809 mov %o2, %o1 ! Remember last tick 5810 brgz,pt %o0, 1b ! Done? 5811 rdpr %tick, %o2 ! Get new tick 5812 5813 retl 5814 nop 5815#else 5816/* This code only works if %tick does not wrap */ 5817 rdpr %tick, %g1 ! Take timer snapshot 5818 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %g2 5819 sethi %hi(MICROPERSEC), %o2 5820 ldx [%g2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %g2 ! Get scale factor 5821 or %o2, %lo(MICROPERSEC), %o2 5822! sethi %hi(_C_LABEL(timerblurb), %o5 ! This is if we plan to tune the clock 5823! ld [%o5 + %lo(_C_LABEL(timerblurb))], %o5 ! with respect to the counter/timer 5824 mulx %o0, %g2, %g2 ! Scale it: (usec * Hz) / 1 x 10^6 = ticks 5825 udivx %g2, %o2, %g2 5826 add %g1, %g2, %g2 5827! add %o5, %g2, %g2 5, %g2, %g2 ! But this gets complicated 5828 rdpr %tick, %g1 ! Top of next itr 5829 mov %g1, %g1 ! Erratum 50 58301: 5831 cmp %g1, %g2 5832 bl,a,pn %xcc, 1b ! Done? 5833 rdpr %tick, %g1 5834 5835 retl 5836 nop 5837#endif 5838 /* 5839 * If something's wrong with the standard setup do this stupid loop 5840 * calibrated for a 143MHz processor. 5841 */ 5842Lstupid_delay: 5843 set 142857143/MICROPERSEC, %o1 5844Lstupid_loop: 5845 brnz,pt %o1, Lstupid_loop 5846 dec %o1 5847 brnz,pt %o0, Lstupid_delay 5848 dec %o0 5849 retl 5850 nop 5851 5852/* 5853 * next_tick(long increment) 5854 * 5855 * Sets the %tick_cmpr register to fire off in `increment' machine 5856 * cycles in the future. Also handles %tick wraparound. In 32-bit 5857 * mode we're limited to a 32-bit increment. 5858 */ 5859ENTRY(next_tick) 5860 rd TICK_CMPR, %o2 5861 rdpr %tick, %o1 5862 5863 mov 1, %o3 ! Mask off high bits of these registers 5864 sllx %o3, 63, %o3 5865 andn %o1, %o3, %o1 5866 andn %o2, %o3, %o2 5867 cmp %o1, %o2 ! Did we wrap? (tick < tick_cmpr) 5868 bgt,pt %icc, 1f 5869 add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. 5870 5871 /* 5872 * Handle the unlikely case of %tick wrapping. 5873 * 5874 * This should only happen every 10 years or more. 5875 * 5876 * We need to increment the time base by the size of %tick in 5877 * microseconds. This will require some divides and multiplies 5878 * which can take time. So we re-read %tick. 5879 * 5880 */ 5881 5882 /* XXXXX NOT IMPLEMENTED */ 5883 5884 5885 58861: 5887 add %o2, %o0, %o2 5888 andn %o2, %o3, %o4 5889 brlz,pn %o4, Ltick_ovflw 5890 cmp %o2, %o1 ! Has this tick passed? 5891 blt,pn %xcc, 1b ! Yes 5892 nop 5893 5894#ifdef BB_ERRATA_1 5895 ba,a 2f 5896 nop 5897#else 5898 retl 5899 wr %o2, TICK_CMPR 5900#endif 5901 5902Ltick_ovflw: 5903/* 5904 * When we get here tick_cmpr has wrapped, but we don't know if %tick 5905 * has wrapped. If bit 62 is set then we have not wrapped and we can 5906 * use the current value of %o4 as %tick. Otherwise we need to return 5907 * to our loop with %o4 as %tick_cmpr (%o2). 5908 */ 5909 srlx %o3, 1, %o5 5910 btst %o5, %o1 5911 bz,pn %xcc, 1b 5912 mov %o4, %o2 5913#ifdef BB_ERRATA_1 5914 ba,a 2f 5915 nop 5916 .align 64 59172: wr %o2, TICK_CMPR 5918 rd TICK_CMPR, %g0 5919 retl 5920 nop 5921#else 5922 retl 5923 wr %o2, TICK_CMPR 5924#endif 5925 5926 5927ENTRY(setjmp) 5928 save %sp, -CC64FSZ, %sp ! Need a frame to return to. 5929 flushw 5930 stx %fp, [%i0+0] ! 64-bit stack pointer 5931 stx %i7, [%i0+8] ! 64-bit return pc 5932 ret 5933 restore %g0, 0, %o0 5934 5935 .data 5936Lpanic_ljmp: 5937 .asciz "longjmp botch" 5938 _ALIGN 5939 .text 5940 5941ENTRY(longjmp) 5942 save %sp, -CC64FSZ, %sp ! prepare to restore to (old) frame 5943 flushw 5944 mov 1, %i2 5945 ldx [%i0+0], %fp ! get return stack 5946 movrz %i1, %i1, %i2 ! compute v ? v : 1 5947 ldx [%i0+8], %i7 ! get rpc 5948 ret 5949 restore %i2, 0, %o0 5950 5951#if defined(DDB) || defined(KGDB) 5952 /* 5953 * Debug stuff. Dump the trap registers into buffer & set tl=0. 5954 * 5955 * %o0 = *ts 5956 */ 5957ENTRY(savetstate) 5958 mov %o0, %o1 5959 rdpr %tl, %o0 5960 brz %o0, 2f 5961 mov %o0, %o2 59621: 5963 rdpr %tstate, %o3 5964 stx %o3, [%o1] 5965 deccc %o2 5966 inc 8, %o1 5967 rdpr %tpc, %o4 5968 stx %o4, [%o1] 5969 inc 8, %o1 5970 rdpr %tnpc, %o5 5971 stx %o5, [%o1] 5972 inc 8, %o1 5973 rdpr %tt, %o4 5974 stx %o4, [%o1] 5975 inc 8, %o1 5976 bnz 1b 5977 wrpr %o2, 0, %tl 59782: 5979 retl 5980 nop 5981 5982 /* 5983 * Debug stuff. Resore trap registers from buffer. 5984 * 5985 * %o0 = %tl 5986 * %o1 = *ts 5987 * 5988 * Maybe this should be re-written to increment tl instead of decrementing. 5989 */ 5990ENTRY(restoretstate) 5991 flushw ! Make sure we don't have stack probs & lose hibits of %o 5992 brz,pn %o0, 2f 5993 mov %o0, %o2 5994 wrpr %o0, 0, %tl 59951: 5996 ldx [%o1], %o3 5997 deccc %o2 5998 inc 8, %o1 5999 wrpr %o3, 0, %tstate 6000 ldx [%o1], %o4 6001 inc 8, %o1 6002 wrpr %o4, 0, %tpc 6003 ldx [%o1], %o5 6004 inc 8, %o1 6005 wrpr %o5, 0, %tnpc 6006 ldx [%o1], %o4 6007 inc 8, %o1 6008 wrpr %o4, 0, %tt 6009 bnz 1b 6010 wrpr %o2, 0, %tl 60112: 6012 retl 6013 wrpr %o0, 0, %tl 6014 6015 /* 6016 * Switch to context in abs(%o0) 6017 */ 6018ENTRY(switchtoctx_us) 6019 set DEMAP_CTX_SECONDARY, %o3 6020 stxa %o3, [%o3] ASI_DMMU_DEMAP 6021 mov CTX_SECONDARY, %o4 6022 stxa %o3, [%o3] ASI_IMMU_DEMAP 6023 membar #Sync 6024 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 6025 sethi %hi(KERNBASE), %o2 6026 membar #Sync 6027 flush %o2 6028 retl 6029 nop 6030 6031ENTRY(switchtoctx_usiii) 6032 mov CTX_SECONDARY, %o4 6033 ldxa [%o4] ASI_DMMU, %o2 ! Load secondary context 6034 mov CTX_PRIMARY, %o5 6035 ldxa [%o5] ASI_DMMU, %o1 ! Save primary context 6036 membar #LoadStore 6037 stxa %o2, [%o5] ASI_DMMU ! Insert secondary for demap 6038 membar #Sync 6039 set DEMAP_CTX_PRIMARY, %o3 6040 stxa %o3, [%o3] ASI_DMMU_DEMAP 6041 membar #Sync 6042 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 6043 membar #Sync 6044 stxa %o1, [%o5] ASI_DMMU ! Restore primary context 6045 sethi %hi(KERNBASE), %o2 6046 membar #Sync 6047 flush %o2 6048 retl 6049 nop 6050 6051#ifndef _LP64 6052 /* 6053 * Convert to 32-bit stack then call OF_sym2val() 6054 */ 6055ENTRY(OF_sym2val32) 6056 save %sp, -CC64FSZ, %sp 6057 btst 7, %i0 6058 bnz,pn %icc, 1f 6059 add %sp, BIAS, %o1 6060 btst 1, %sp 6061 movnz %icc, %o1, %sp 6062 call _C_LABEL(OF_sym2val) 6063 mov %i0, %o0 60641: 6065 ret 6066 restore %o0, 0, %o0 6067 6068 /* 6069 * Convert to 32-bit stack then call OF_val2sym() 6070 */ 6071ENTRY(OF_val2sym32) 6072 save %sp, -CC64FSZ, %sp 6073 btst 7, %i0 6074 bnz,pn %icc, 1f 6075 add %sp, BIAS, %o1 6076 btst 1, %sp 6077 movnz %icc, %o1, %sp 6078 call _C_LABEL(OF_val2sym) 6079 mov %i0, %o0 60801: 6081 ret 6082 restore %o0, 0, %o0 6083#endif /* _LP64 */ 6084#endif /* DDB */ 6085 6086 .data 6087 _ALIGN 6088#if NKSYMS || defined(DDB) || defined(LKM) 6089 .globl _C_LABEL(esym) 6090_C_LABEL(esym): 6091 POINTER 0 6092 .globl _C_LABEL(ssym) 6093_C_LABEL(ssym): 6094 POINTER 0 6095#endif 6096 .comm _C_LABEL(promvec), PTRSZ 6097 6098#ifdef DEBUG 6099 .comm _C_LABEL(trapdebug), 4 6100 .comm _C_LABEL(pmapdebug), 4 6101#endif 6102