xref: /illumos-gate/usr/src/uts/sun4v/vm/mach_sfmmu_asm.S (revision 55fea89d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * SFMMU primitives.  These primitives should only be used by sfmmu
 * routines.
 */

#include "assym.h"

#include <sys/asm_linkage.h>
#include <sys/machtrap.h>
#include <sys/machasi.h>
#include <sys/sun4asi.h>
#include <sys/pte.h>
#include <sys/mmu.h>
#include <vm/hat_sfmmu.h>
#include <vm/seg_spt.h>
#include <sys/machparam.h>
#include <sys/privregs.h>
#include <sys/scb.h>
#include <sys/intreg.h>
#include <sys/machthread.h>
#include <sys/clock.h>
#include <sys/trapstat.h>

/*
 * sfmmu related subroutines
 */

/*
 * Invalidate either the context of a specific victim or any process
 * currently running on this CPU.
 *
 * %g1 = sfmmup whose ctx is being stolen (victim)
 *	 when called from sfmmu_wrap_around, %g1 == INVALID_CONTEXT.
 * Note %g1 is the only input argument used by this xcall handler.
 */

	ENTRY(sfmmu_raise_tsb_exception)
	!
	! if (victim == INVALID_CONTEXT) {
	!	if (sec-ctx > INVALID_CONTEXT)
	!		write INVALID_CONTEXT to sec-ctx
	!	if (pri-ctx > INVALID_CONTEXT)
	!		write INVALID_CONTEXT to pri-ctx
	!
	! } else if (current CPU tsbmiss->usfmmup != victim sfmmup) {
	!	return
	! } else {
	!	if (sec-ctx > INVALID_CONTEXT)
	!		write INVALID_CONTEXT to sec-ctx
	!
	!	if (pri-ctx > INVALID_CONTEXT)
	!		write INVALID_CONTEXT to pri-ctx
	! }
	!

	sethi   %hi(ksfmmup), %g3
	ldx	[%g3 + %lo(ksfmmup)], %g3
	cmp	%g1, %g3
	be,a,pn %xcc, ptl1_panic	/* can't invalidate kernel ctx */
	  mov	PTL1_BAD_RAISE_TSBEXCP, %g1

	set	INVALID_CONTEXT, %g2

	cmp	%g1, INVALID_CONTEXT
	bne,pt	%xcc, 1f			/* called from wrap_around? */
	  mov	MMU_SCONTEXT, %g3

	ldxa	[%g3]ASI_MMU_CTX, %g5		/* %g5 = sec-ctx */
	cmp	%g5, INVALID_CONTEXT		/* kernel  or invalid ctx ? */
	ble,pn	%xcc, 0f			/* yes, no need to change */
	  mov	MMU_PCONTEXT, %g7

	stxa	%g2, [%g3]ASI_MMU_CTX		/* set invalid ctx */
	membar	#Sync

0:
	ldxa	[%g7]ASI_MMU_CTX, %g5		/* %g5 = pri-ctx */
	cmp	%g5, INVALID_CONTEXT		/* kernel or invalid ctx? */
	ble,pn	%xcc, 6f			/* yes, no need to change */
	  nop

	stxa	%g2, [%g7]ASI_MMU_CTX		/* set pri-ctx to invalid  */
	membar	#Sync

6:	/* flushall tlb */
	mov	%o0, %g3
	mov	%o1, %g4
	mov	%o2, %g6
	mov	%o5, %g7

        mov     %g0, %o0        ! XXX no cpu list yet
        mov     %g0, %o1        ! XXX no cpu list yet
        mov     MAP_ITLB | MAP_DTLB, %o2
        mov     MMU_DEMAP_ALL, %o5
        ta      FAST_TRAP
        brz,pt  %o0, 5f
          nop
     	ba ptl1_panic		/* bad HV call */
	  mov	PTL1_BAD_RAISE_TSBEXCP, %g1
5:
	mov	%g3, %o0
	mov	%g4, %o1
	mov	%g6, %o2
	mov	%g7, %o5

	ba	3f
	  nop
1:
	/*
	 * %g1 = sfmmup
	 * %g2 = INVALID_CONTEXT
	 * %g3 = MMU_SCONTEXT
	 */
	CPU_TSBMISS_AREA(%g5, %g6)		/* load cpu tsbmiss area */
	ldx	[%g5 + TSBMISS_UHATID], %g5     /* load usfmmup */

	cmp	%g5, %g1			/* is it the victim? */
	bne,pt	%xcc, 2f			/* is our sec-ctx a victim? */
	  nop

	ldxa    [%g3]ASI_MMU_CTX, %g5           /* %g5 = sec-ctx */
	cmp     %g5, INVALID_CONTEXT            /* kernel  or invalid ctx ? */
	ble,pn  %xcc, 0f                        /* yes, no need to change */
	  mov	MMU_PCONTEXT, %g7

	stxa	%g2, [%g3]ASI_MMU_CTX		/* set sec-ctx to invalid */
	membar	#Sync

0:
	ldxa	[%g7]ASI_MMU_CTX, %g4		/* %g4 = pri-ctx */
	cmp	%g4, INVALID_CONTEXT		/* is pri-ctx the victim? */
	ble 	%icc, 3f			/* no need to change pri-ctx */
	  nop
	stxa	%g2, [%g7]ASI_MMU_CTX		/* set pri-ctx to invalid  */
	membar	#Sync

3:
	/* TSB program must be cleared - walkers do not check a context. */
	mov	%o0, %g3
	mov	%o1, %g4
	mov	%o5, %g7
	clr	%o0
	clr	%o1
	mov	MMU_TSB_CTXNON0, %o5
	ta	FAST_TRAP
	brnz,a,pn %o0, ptl1_panic
	  mov	PTL1_BAD_HCALL, %g1
	mov	%g3, %o0
	mov	%g4, %o1
	mov	%g7, %o5
2:
	retry
	SET_SIZE(sfmmu_raise_tsb_exception)

	ENTRY_NP(sfmmu_getctx_pri)
	set	MMU_PCONTEXT, %o0
	retl
	ldxa	[%o0]ASI_MMU_CTX, %o0
	SET_SIZE(sfmmu_getctx_pri)

	ENTRY_NP(sfmmu_getctx_sec)
	set	MMU_SCONTEXT, %o0
	retl
	ldxa	[%o0]ASI_MMU_CTX, %o0
	SET_SIZE(sfmmu_getctx_sec)

	/*
	 * Set the secondary context register for this process.
	 * %o0 = context number
	 */
	ENTRY_NP(sfmmu_setctx_sec)
	/*
	 * From resume we call sfmmu_setctx_sec with interrupts disabled.
	 * But we can also get called from C with interrupts enabled. So,
	 * we need to check first.
	 */

	/* If interrupts are not disabled, then disable them */
	rdpr	%pstate, %g1
	btst	PSTATE_IE, %g1
	bnz,a,pt %icc, 1f
	wrpr	%g1, PSTATE_IE, %pstate		/* disable interrupts */
1:
	mov	MMU_SCONTEXT, %o1
	stxa	%o0, [%o1]ASI_MMU_CTX		/* set 2nd context reg. */
	membar	#Sync
        /*
         * if the routine is entered with intr enabled, then enable intr now.
         * otherwise, keep intr disabled, return without enabing intr.
         * %g1 - old intr state
         */
        btst    PSTATE_IE, %g1
        bnz,a,pt %icc, 2f
        wrpr    %g0, %g1, %pstate               /* enable interrupts */
2:      retl
        nop
        SET_SIZE(sfmmu_setctx_sec)

	/*
	 * set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS.
	 * returns the detection value in %o0.
	 */
	ENTRY_NP(sfmmu_setup_4lp)
	set	ktsb_phys, %o2
	mov	1, %o1
	st	%o1, [%o2]
	retl
	mov	%o1, %o0
	SET_SIZE(sfmmu_setup_4lp)

	/*
	 * Called to load MMU registers and tsbmiss area
	 * for the active process.  This function should
	 * only be called from TL=0.
	 *
	 * %o0 - hat pointer
	 */
	ENTRY_NP(sfmmu_load_mmustate)

#ifdef DEBUG
	PANIC_IF_INTR_ENABLED_PSTR(msfmmu_ei_l1, %g1)
#endif /* DEBUG */

	sethi	%hi(ksfmmup), %o3
	ldx	[%o3 + %lo(ksfmmup)], %o3
	cmp	%o3, %o0
	be,pn	%xcc, 7f			! if kernel as, do nothing
	  nop

	set     MMU_SCONTEXT, %o3
        ldxa    [%o3]ASI_MMU_CTX, %o5

	cmp	%o5, INVALID_CONTEXT		! ctx is invalid?
	bne,pt	%icc, 1f
	  nop

	CPU_TSBMISS_AREA(%o2, %o3)		! %o2 = tsbmiss area
	stx	%o0, [%o2 + TSBMISS_UHATID]
	stx	%g0, [%o2 +  TSBMISS_SHARED_UHATID]
#ifdef DEBUG
	/* check if hypervisor/hardware should handle user TSB */
	sethi	%hi(hv_use_non0_tsb), %o2
	ld	[%o2 + %lo(hv_use_non0_tsb)], %o2
	brz,pn	%o2, 0f
	  nop
#endif /* DEBUG */
	clr	%o0				! ntsb = 0 for invalid ctx
	clr	%o1				! HV_TSB_INFO_PA = 0 if inv ctx
	mov	MMU_TSB_CTXNON0, %o5
	ta	FAST_TRAP			! set TSB info for user process
	brnz,a,pn %o0, panic_bad_hcall
	  mov	MMU_TSB_CTXNON0, %o1
0:
	retl
	  nop
1:
	/*
	 * We need to set up the TSB base register, tsbmiss
	 * area, and pass the TSB information into the hypervisor
	 */
	ldx	[%o0 + SFMMU_TSB], %o1		! %o1 = first tsbinfo
	ldx	[%o1 + TSBINFO_NEXTPTR], %g2	! %g2 = second tsbinfo

	/* create/set first UTSBREG */
	MAKE_UTSBREG(%o1, %o2, %o3)		! %o2 = user tsbreg
	SET_UTSBREG(SCRATCHPAD_UTSBREG1, %o2, %o3)

	brz,pt	%g2, 2f
	  mov	-1, %o2				! use -1 if no second TSB

	/* make 2nd UTSBREG */
	MAKE_UTSBREG(%g2, %o2, %o3)		! %o2 = user tsbreg
2:
	SET_UTSBREG(SCRATCHPAD_UTSBREG2, %o2, %o3)

        /* make 3rd and 4th TSB */
	CPU_TSBMISS_AREA(%o4, %o3)		! %o4 = tsbmiss area

	ldx	[%o0 + SFMMU_SCDP], %g2		! %g2 = sfmmu_scd
	brz,pt	%g2, 3f
	  mov	-1, %o2				! use -1 if no third TSB

	ldx	[%g2 + SCD_SFMMUP], %g3		! %g3 = scdp->scd_sfmmup
	ldx	[%g3 + SFMMU_TSB], %o1		! %o1 = first scd tsbinfo
	brz,pn %o1, 9f
	  nop					! panic if no third TSB

	/* make 3rd UTSBREG */
	MAKE_UTSBREG(%o1, %o2, %o3)		! %o2 = user tsbreg
3:
	SET_UTSBREG_SHCTX(%o4, TSBMISS_TSBSCDPTR, %o2)

	brz,pt	%g2, 4f
	  mov	-1, %o2				! use -1 if no 3rd or 4th TSB

	brz,pt	%o1, 4f
	  mov	-1, %o2				! use -1 if no 3rd or 4th TSB
	ldx	[%o1 + TSBINFO_NEXTPTR], %g2	! %g2 = second scd tsbinfo
	brz,pt	%g2, 4f
	  mov	-1, %o2				! use -1 if no 4th TSB

	/* make 4th UTSBREG */
	MAKE_UTSBREG(%g2, %o2, %o3)		! %o2 = user tsbreg
4:
	SET_UTSBREG_SHCTX(%o4, TSBMISS_TSBSCDPTR4M, %o2)

#ifdef DEBUG
	/* check if hypervisor/hardware should handle user TSB */
	sethi	%hi(hv_use_non0_tsb), %o2
	ld	[%o2 + %lo(hv_use_non0_tsb)], %o2
	brz,pn	%o2, 6f
	  nop
#endif /* DEBUG */
	CPU_ADDR(%o2, %o4)	! load CPU struct addr to %o2 using %o4
	ldub    [%o2 + CPU_TSTAT_FLAGS], %o1	! load cpu_tstat_flag to %o1

	mov	%o0, %o3			! preserve %o0
	btst	TSTAT_TLB_STATS, %o1
	bnz,a,pn %icc, 5f			! ntsb = 0 if TLB stats enabled
	  clr	%o0

	ldx	[%o3 + SFMMU_HVBLOCK + HV_TSB_INFO_CNT], %o0
5:
	ldx	[%o3 + SFMMU_HVBLOCK + HV_TSB_INFO_PA], %o1
	mov	MMU_TSB_CTXNON0, %o5
	ta	FAST_TRAP			! set TSB info for user process
	brnz,a,pn %o0, panic_bad_hcall
	mov	MMU_TSB_CTXNON0, %o1
	mov	%o3, %o0			! restore %o0
6:
	ldx	[%o0 + SFMMU_ISMBLKPA], %o1	! copy members of sfmmu
	CPU_TSBMISS_AREA(%o2, %o3)		! %o2 = tsbmiss area
	stx	%o1, [%o2 + TSBMISS_ISMBLKPA]	! sfmmu_tsb_miss into the
	ldub	[%o0 + SFMMU_TTEFLAGS], %o3	! per-CPU tsbmiss area.
	ldub	[%o0 + SFMMU_RTTEFLAGS], %o4
	ldx	[%o0 + SFMMU_SRDP], %o1
	stx	%o0, [%o2 + TSBMISS_UHATID]
	stub	%o3, [%o2 + TSBMISS_UTTEFLAGS]
	stub	%o4,  [%o2 + TSBMISS_URTTEFLAGS]
	stx	%o1, [%o2 +  TSBMISS_SHARED_UHATID]
	brz,pn	%o1, 7f				! check for sfmmu_srdp
	  add	%o0, SFMMU_HMERMAP, %o1
	add	%o2, TSBMISS_SHMERMAP, %o2
	mov	SFMMU_HMERGNMAP_WORDS, %o3
						! set tsbmiss shmermap
	SET_REGION_MAP(%o1, %o2, %o3, %o4, load_shme_mmustate)

	ldx	[%o0 + SFMMU_SCDP], %o4		! %o4 = sfmmu_scd
	CPU_TSBMISS_AREA(%o2, %o3)		! %o2 = tsbmiss area
	mov	SFMMU_HMERGNMAP_WORDS, %o3
	brnz,pt	%o4, 8f				! check for sfmmu_scdp else
	  add	%o2, TSBMISS_SCDSHMERMAP, %o2	! zero tsbmiss scd_shmermap
	ZERO_REGION_MAP(%o2, %o3, zero_scd_mmustate)
7:
	retl
	nop
8:						! set tsbmiss scd_shmermap
	add	%o4, SCD_HMERMAP, %o1
	SET_REGION_MAP(%o1, %o2, %o3, %o4, load_scd_mmustate)
	retl
	  nop
9:
	sethi   %hi(panicstr), %g1		! panic if no 3rd TSB
        ldx     [%g1 + %lo(panicstr)], %g1
        tst     %g1

        bnz,pn  %xcc, 7b
          nop

        sethi   %hi(sfmmu_panic10), %o0
        call    panic
          or      %o0, %lo(sfmmu_panic10), %o0

	SET_SIZE(sfmmu_load_mmustate)

	ENTRY(prefetch_tsbe_read)
	retl
	nop
	SET_SIZE(prefetch_tsbe_read)

	ENTRY(prefetch_tsbe_write)
	retl
	nop
	SET_SIZE(prefetch_tsbe_write)