1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/*- 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include "assym.inc" 82#include "opt_sched.h" 83 84#include <machine/asm.h> 85#include <machine/asmacros.h> 86#include <machine/armreg.h> 87#include <machine/sysreg.h> 88#include <machine/vfp.h> 89 90__FBSDID("$FreeBSD$"); 91 92#if defined(SMP) 93#define GET_PCPU(tmp, tmp2) \ 94 mrc CP15_MPIDR(tmp); \ 95 and tmp, tmp, #0xf; \ 96 ldr tmp2, .Lcurpcpu+4; \ 97 mul tmp, tmp, tmp2; \ 98 ldr tmp2, .Lcurpcpu; \ 99 add tmp, tmp, tmp2; 100#else 101 102#define GET_PCPU(tmp, tmp2) \ 103 ldr tmp, .Lcurpcpu 104#endif 105 106#ifdef VFP 107 .fpu vfp /* allow VFP instructions */ 108#endif 109 110.Lcurpcpu: 111 .word _C_LABEL(__pcpu) 112 .word PCPU_SIZE 113.Lblocked_lock: 114 .word _C_LABEL(blocked_lock) 115 116ENTRY(cpu_context_switch) 117 DSB 118 /* 119 * We can directly switch between translation tables only when the 120 * size of the mapping for any given virtual address is the same 121 * in the old and new translation tables. 122 * Thus, we must switch to kernel pmap translation table as 123 * intermediate mapping because all sizes of these mappings are same 124 * (or unmapped). The same is true for switch from kernel pmap 125 * translation table to new pmap one. 126 */ 127 mov r2, #(CPU_ASID_KERNEL) 128 ldr r1, =(_C_LABEL(pmap_kern_ttb)) 129 ldr r1, [r1] 130 mcr CP15_TTBR0(r1) /* switch to kernel TTB */ 131 ISB 132 mcr CP15_TLBIASID(r2) /* flush not global TLBs */ 133 DSB 134 mcr CP15_TTBR0(r0) /* switch to new TTB */ 135 ISB 136 /* 137 * We must flush not global TLBs again because PT2MAP mapping 138 * is different. 139 */ 140 mcr CP15_TLBIASID(r2) /* flush not global TLBs */ 141 /* 142 * Flush entire Branch Target Cache because of the branch predictor 143 * is not architecturally invisible. See ARM Architecture Reference 144 * Manual ARMv7-A and ARMv7-R edition, page B2-1264(65), Branch 145 * predictors and Requirements for branch predictor maintenance 146 * operations sections. 147 */ 148 /* 149 * Additionally, to mitigate mistrained branch predictor attack 150 * we must invalidate it on affected CPUs. Unfortunately, BPIALL 151 * is effectively NOP on Cortex-A15 so it needs special treatment. 152 */ 153 ldr r0, [r8, #PC_BP_HARDEN_KIND] 154 cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU 155 mcrne CP15_BPIALL /* Flush entire Branch Target Cache */ 156 mcreq CP15_ICIALLU /* This is the only way how to flush */ 157 /* Branch Target Cache on Cortex-A15. */ 158 DSB 159 mov pc, lr 160END(cpu_context_switch) 161 162/* 163 * cpu_throw(oldtd, newtd) 164 * 165 * Remove current thread state, then select the next thread to run 166 * and load its state. 167 * r0 = oldtd 168 * r1 = newtd 169 */ 170ENTRY(cpu_throw) 171 mov r10, r0 /* r10 = oldtd */ 172 mov r11, r1 /* r11 = newtd */ 173 174#ifdef VFP /* This thread is dying, disable */ 175 bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ 176#endif 177 GET_PCPU(r8, r9) /* r8 = current pcpu */ 178 ldr r4, [r8, #PC_CPUID] /* r4 = current cpu id */ 179 180 cmp r10, #0 /* old thread? */ 181 beq 2f /* no, skip */ 182 183 /* Remove this CPU from the active list. */ 184 ldr r5, [r8, #PC_CURPMAP] 185 mov r0, #(PM_ACTIVE) 186 add r5, r0 /* r5 = old pm_active */ 187 188 /* Compute position and mask. */ 189#if _NCPUWORDS > 1 190 lsr r0, r4, #3 191 bic r0, #3 192 add r5, r0 /* r5 = position in old pm_active */ 193 mov r2, #1 194 and r0, r4, #31 195 lsl r2, r0 /* r2 = mask */ 196#else 197 mov r2, #1 198 lsl r2, r4 /* r2 = mask */ 199#endif 200 /* Clear cpu from old active list. */ 201#ifdef SMP 2021: ldrex r0, [r5] 203 bic r0, r2 204 strex r1, r0, [r5] 205 teq r1, #0 206 bne 1b 207#else 208 ldr r0, [r5] 209 bic r0, r2 210 str r0, [r5] 211#endif 212 2132: 214#ifdef INVARIANTS 215 cmp r11, #0 /* new thread? */ 216 beq badsw1 /* no, panic */ 217#endif 218 ldr r7, [r11, #(TD_PCB)] /* r7 = new PCB */ 219 220 /* 221 * Registers at this point 222 * r4 = current cpu id 223 * r7 = new PCB 224 * r8 = current pcpu 225 * r11 = newtd 226 */ 227 228 /* MMU switch to new thread. */ 229 ldr r0, [r7, #(PCB_PAGEDIR)] 230#ifdef INVARIANTS 231 cmp r0, #0 /* new thread? */ 232 beq badsw4 /* no, panic */ 233#endif 234 bl _C_LABEL(cpu_context_switch) 235 236 /* 237 * Set new PMAP as current one. 238 * Insert cpu to new active list. 239 */ 240 241 ldr r6, [r11, #(TD_PROC)] /* newtd->proc */ 242 ldr r6, [r6, #(P_VMSPACE)] /* newtd->proc->vmspace */ 243 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ 244 str r6, [r8, #PC_CURPMAP] /* store to curpmap */ 245 246 mov r0, #PM_ACTIVE 247 add r6, r0 /* r6 = new pm_active */ 248 249 /* compute position and mask */ 250#if _NCPUWORDS > 1 251 lsr r0, r4, #3 252 bic r0, #3 253 add r6, r0 /* r6 = position in new pm_active */ 254 mov r2, #1 255 and r0, r4, #31 256 lsl r2, r0 /* r2 = mask */ 257#else 258 mov r2, #1 259 lsl r2, r4 /* r2 = mask */ 260#endif 261 /* Set cpu to new active list. */ 262#ifdef SMP 2631: ldrex r0, [r6] 264 orr r0, r2 265 strex r1, r0, [r6] 266 teq r1, #0 267 bne 1b 268#else 269 ldr r0, [r6] 270 orr r0, r2 271 str r0, [r6] 272#endif 273 /* 274 * Registers at this point. 275 * r7 = new PCB 276 * r8 = current pcpu 277 * r11 = newtd 278 * They must match the ones in sw1 position !!! 279 */ 280 DMB 281 b sw1 /* share new thread init with cpu_switch() */ 282END(cpu_throw) 283 284/* 285 * cpu_switch(oldtd, newtd, lock) 286 * 287 * Save the current thread state, then select the next thread to run 288 * and load its state. 289 * r0 = oldtd 290 * r1 = newtd 291 * r2 = lock (new lock for old thread) 292 */ 293ENTRY(cpu_switch) 294 /* Interrupts are disabled. */ 295#ifdef INVARIANTS 296 cmp r0, #0 /* old thread? */ 297 beq badsw2 /* no, panic */ 298#endif 299 /* Save all the registers in the old thread's pcb. */ 300 ldr r3, [r0, #(TD_PCB)] 301 add r3, #(PCB_R4) 302 stmia r3, {r4-r12, sp, lr, pc} 303 mrc CP15_TPIDRURW(r4) 304 str r4, [r3, #(PCB_TPIDRURW - PCB_R4)] 305 306#ifdef INVARIANTS 307 cmp r1, #0 /* new thread? */ 308 beq badsw3 /* no, panic */ 309#endif 310 /* 311 * Save arguments. Note that we can now use r0-r14 until 312 * it is time to restore them for the new thread. However, 313 * some registers are not safe over function call. 314 */ 315 mov r9, r2 /* r9 = lock */ 316 mov r10, r0 /* r10 = oldtd */ 317 mov r11, r1 /* r11 = newtd */ 318 319 GET_PCPU(r8, r3) /* r8 = current PCPU */ 320 ldr r7, [r11, #(TD_PCB)] /* r7 = newtd->td_pcb */ 321 322 323 324#ifdef VFP 325 ldr r3, [r10, #(TD_PCB)] 326 mov r1, r3 327 mov r0, r10 328 bl _C_LABEL(vfp_save_state) 329#endif 330 331 /* 332 * MMU switch. If we're switching to a thread with the same 333 * address space as the outgoing one, we can skip the MMU switch. 334 */ 335 mrc CP15_TTBR0(r1) /* r1 = old TTB */ 336 ldr r0, [r7, #(PCB_PAGEDIR)] /* r0 = new TTB */ 337 cmp r0, r1 /* Switching to the TTB? */ 338 beq sw0 /* same TTB, skip */ 339 340#ifdef INVARIANTS 341 cmp r0, #0 /* new thread? */ 342 beq badsw4 /* no, panic */ 343#endif 344 345 bl cpu_context_switch /* new TTB as argument */ 346 347 /* 348 * Registers at this point 349 * r7 = new PCB 350 * r8 = current pcpu 351 * r9 = lock 352 * r10 = oldtd 353 * r11 = newtd 354 */ 355 356 /* 357 * Set new PMAP as current one. 358 * Update active list on PMAPs. 359 */ 360 ldr r6, [r11, #TD_PROC] /* newtd->proc */ 361 ldr r6, [r6, #P_VMSPACE] /* newtd->proc->vmspace */ 362 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ 363 364 ldr r5, [r8, #PC_CURPMAP] /* get old curpmap */ 365 str r6, [r8, #PC_CURPMAP] /* and save new one */ 366 367 mov r0, #PM_ACTIVE 368 add r5, r0 /* r5 = old pm_active */ 369 add r6, r0 /* r6 = new pm_active */ 370 371 /* Compute position and mask. */ 372 ldr r4, [r8, #PC_CPUID] 373#if _NCPUWORDS > 1 374 lsr r0, r4, #3 375 bic r0, #3 376 add r5, r0 /* r5 = position in old pm_active */ 377 add r6, r0 /* r6 = position in new pm_active */ 378 mov r2, #1 379 and r0, r4, #31 380 lsl r2, r0 /* r2 = mask */ 381#else 382 mov r2, #1 383 lsl r2, r4 /* r2 = mask */ 384#endif 385 /* Clear cpu from old active list. */ 386#ifdef SMP 3871: ldrex r0, [r5] 388 bic r0, r2 389 strex r1, r0, [r5] 390 teq r1, #0 391 bne 1b 392#else 393 ldr r0, [r5] 394 bic r0, r2 395 str r0, [r5] 396#endif 397 /* Set cpu to new active list. */ 398#ifdef SMP 3991: ldrex r0, [r6] 400 orr r0, r2 401 strex r1, r0, [r6] 402 teq r1, #0 403 bne 1b 404#else 405 ldr r0, [r6] 406 orr r0, r2 407 str r0, [r6] 408#endif 409 410sw0: 411 /* 412 * Registers at this point 413 * r7 = new PCB 414 * r8 = current pcpu 415 * r9 = lock 416 * r10 = oldtd 417 * r11 = newtd 418 */ 419 420 /* Change the old thread lock. */ 421 add r5, r10, #TD_LOCK 422 DMB 4231: ldrex r0, [r5] 424 strex r1, r9, [r5] 425 teq r1, #0 426 bne 1b 427 DMB 428 429sw1: 430 clrex 431 /* 432 * Registers at this point 433 * r7 = new PCB 434 * r8 = current pcpu 435 * r11 = newtd 436 */ 437 438#if defined(SMP) && defined(SCHED_ULE) 439 /* 440 * 386 and amd64 do the blocked lock test only for SMP and SCHED_ULE 441 * QQQ: What does it mean in reality and why is it done? 442 */ 443 ldr r6, =blocked_lock 4441: 445 ldr r3, [r11, #TD_LOCK] /* atomic write regular read */ 446 cmp r3, r6 447 beq 1b 448#endif 449 450 /* We have a new curthread now so make a note it */ 451 str r11, [r8, #PC_CURTHREAD] 452 mcr CP15_TPIDRPRW(r11) 453 454 /* store pcb in per cpu structure */ 455 str r7, [r8, #PC_CURPCB] 456 457 /* 458 * Restore all saved registers and return. Note that some saved 459 * registers can be changed when either cpu_fork(), cpu_copy_thread(), 460 * cpu_fork_kthread_handler(), or makectx() was called. 461 * 462 * The value of TPIDRURW is also written into TPIDRURO, as 463 * userspace still uses TPIDRURO, modifying it through 464 * sysarch(ARM_SET_TP, addr). 465 */ 466 ldr r3, [r7, #PCB_TPIDRURW] 467 mcr CP15_TPIDRURW(r3) /* write tls thread reg 2 */ 468 mcr CP15_TPIDRURO(r3) /* write tls thread reg 3 */ 469 add r3, r7, #PCB_R4 470 ldmia r3, {r4-r12, sp, pc} 471 472#ifdef INVARIANTS 473badsw1: 474 ldr r0, =sw1_panic_str 475 bl _C_LABEL(panic) 4761: nop 477 b 1b 478 479badsw2: 480 ldr r0, =sw2_panic_str 481 bl _C_LABEL(panic) 4821: nop 483 b 1b 484 485badsw3: 486 ldr r0, =sw3_panic_str 487 bl _C_LABEL(panic) 4881: nop 489 b 1b 490 491badsw4: 492 ldr r0, =sw4_panic_str 493 bl _C_LABEL(panic) 4941: nop 495 b 1b 496 497sw1_panic_str: 498 .asciz "cpu_throw: no newthread supplied.\n" 499sw2_panic_str: 500 .asciz "cpu_switch: no curthread supplied.\n" 501sw3_panic_str: 502 .asciz "cpu_switch: no newthread supplied.\n" 503sw4_panic_str: 504 .asciz "cpu_switch: new pagedir is NULL.\n" 505#endif 506END(cpu_switch) 507