1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/*- 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include "assym.s" 82 83#include <machine/asm.h> 84#include <machine/asmacros.h> 85#include <machine/armreg.h> 86__FBSDID("$FreeBSD$"); 87 88 89/* 90 * New experimental definitions of IRQdisable and IRQenable 91 * These keep FIQ's enabled since FIQ's are special. 92 */ 93 94#define DOMAIN_CLIENT 0x01 95#define IRQdisable \ 96 mrs r14, cpsr ; \ 97 orr r14, r14, #(I32_bit) ; \ 98 msr cpsr_c, r14 ; \ 99 100#define IRQenable \ 101 mrs r14, cpsr ; \ 102 bic r14, r14, #(I32_bit) ; \ 103 msr cpsr_c, r14 ; \ 104 105/* 106 * These are used for switching the translation table/DACR. 107 * Since the vector page can be invalid for a short time, we must 108 * disable both regular IRQs *and* FIQs. 109 * 110 * XXX: This is not necessary if the vector table is relocated. 111 */ 112#define IRQdisableALL \ 113 mrs r14, cpsr ; \ 114 orr r14, r14, #(I32_bit | F32_bit) ; \ 115 msr cpsr_c, r14 116 117#define IRQenableALL \ 118 mrs r14, cpsr ; \ 119 bic r14, r14, #(I32_bit | F32_bit) ; \ 120 msr cpsr_c, r14 121 122.Lcurpcb: 123 .word _C_LABEL(__pcpu) + PC_CURPCB 124.Lcpufuncs: 125 .word _C_LABEL(cpufuncs) 126.Lblock_userspace_access: 127 .word _C_LABEL(block_userspace_access) 128.Lcpu_do_powersave: 129 .word _C_LABEL(cpu_do_powersave) 130.Lblocked_lock: 131 .word _C_LABEL(blocked_lock) 132ENTRY(cpu_throw) 133 mov r5, r1 134 135 /* 136 * r5 = newtd 137 */ 138 139 ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ 140 141 /* Switch to lwp0 context */ 142 143 ldr r9, .Lcpufuncs 144 mov lr, pc 145 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 146 ldr r0, [r7, #(PCB_PL1VEC)] 147 ldr r1, [r7, #(PCB_DACR)] 148 /* 149 * r0 = Pointer to L1 slot for vector_page (or NULL) 150 * r1 = lwp0's DACR 151 * r5 = lwp0 152 * r6 = exit func 153 * r7 = lwp0's PCB 154 * r9 = cpufuncs 155 */ 156 157 /* 158 * Ensure the vector table is accessible by fixing up lwp0's L1 159 */ 160 cmp r0, #0 /* No need to fixup vector table? */ 161 ldrne r3, [r0] /* But if yes, fetch current value */ 162 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 163 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 164 cmpne r3, r2 /* Stuffing the same value? */ 165 strne r2, [r0] /* Store if not. */ 166 167#ifdef PMAP_INCLUDE_PTE_SYNC 168 /* 169 * Need to sync the cache to make sure that last store is 170 * visible to the MMU. 171 */ 172 movne r1, #4 173 movne lr, pc 174 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 175#endif /* PMAP_INCLUDE_PTE_SYNC */ 176 177 /* 178 * Note: We don't do the same optimisation as cpu_switch() with 179 * respect to avoiding flushing the TLB if we're switching to 180 * the same L1 since this process' VM space may be about to go 181 * away, so we don't want *any* turds left in the TLB. 182 */ 183 184 /* Switch the memory to the new process */ 185 ldr r0, [r7, #(PCB_PAGEDIR)] 186 mov lr, pc 187 ldr pc, [r9, #CF_CONTEXT_SWITCH] 188 189 /* Restore all the save registers */ 190#ifndef _ARM_ARCH_5E 191 add r1, r7, #PCB_R8 192 ldmia r1, {r8-r13} 193#else 194 ldr r8, [r7, #(PCB_R8)] 195 ldr r9, [r7, #(PCB_R9)] 196 ldr r10, [r7, #(PCB_R10)] 197 ldr r11, [r7, #(PCB_R11)] 198 ldr r12, [r7, #(PCB_R12)] 199 ldr r13, [r7, #(PCB_SP)] 200#endif 201 202 /* We have a new curthread now so make a note it */ 203 ldr r6, .Lcurthread 204 str r5, [r6] 205 206 /* Set the new tp */ 207 ldr r6, [r5, #(TD_MD + MD_TP)] 208 ldr r4, =ARM_TP_ADDRESS 209 str r6, [r4] 210 ldr r6, [r5, #(TD_MD + MD_RAS_START)] 211 str r6, [r4, #4] /* ARM_RAS_START */ 212 ldr r6, [r5, #(TD_MD + MD_RAS_END)] 213 str r6, [r4, #8] /* ARM_RAS_END */ 214 215 /* Hook in a new pcb */ 216 ldr r6, .Lcurpcb 217 str r7, [r6] 218 219 ldmfd sp!, {r4-r7, pc} 220 221ENTRY(cpu_switch) 222 stmfd sp!, {r4-r7, lr} 223 mov r6, r2 /* Save the mutex */ 224 225.Lswitch_resume: 226 /* rem: r0 = old lwp */ 227 /* rem: interrupts are disabled */ 228 229#ifdef MULTIPROCESSOR 230 /* XXX use curcpu() */ 231 ldr r2, .Lcpu_info_store 232 str r2, [r6, #(L_CPU)] 233#endif 234 235 /* Process is now on a processor. */ 236 237 /* We have a new curthread now so make a note it */ 238 ldr r7, .Lcurthread 239 str r1, [r7] 240 241 /* Hook in a new pcb */ 242 ldr r7, .Lcurpcb 243 ldr r2, [r1, #TD_PCB] 244 str r2, [r7] 245 246 /* rem: r1 = new process */ 247 /* rem: interrupts are enabled */ 248 249 /* Stage two : Save old context */ 250 251 /* Get the user structure for the old thread. */ 252 ldr r2, [r0, #(TD_PCB)] 253 mov r4, r0 /* Save the old thread. */ 254 255 /* Save all the registers in the old thread's pcb */ 256#ifndef _ARM_ARCH_5E 257 add r7, r2, #(PCB_R8) 258 stmia r7, {r8-r13} 259#else 260 strd r8, [r2, #(PCB_R8)] 261 strd r10, [r2, #(PCB_R10)] 262 strd r12, [r2, #(PCB_R12)] 263#endif 264 str pc, [r2, #(PCB_PC)] 265 266 /* 267 * NOTE: We can now use r8-r13 until it is time to restore 268 * them for the new process. 269 */ 270 /* Store the old tp */ 271 ldr r3, =ARM_TP_ADDRESS 272 ldr r9, [r3] 273 str r9, [r0, #(TD_MD + MD_TP)] 274 ldr r9, [r3, #4] 275 str r9, [r0, #(TD_MD + MD_RAS_START)] 276 ldr r9, [r3, #8] 277 str r9, [r0, #(TD_MD + MD_RAS_END)] 278 279 /* Set the new tp */ 280 ldr r9, [r1, #(TD_MD + MD_TP)] 281 str r9, [r3] 282 ldr r9, [r1, #(TD_MD + MD_RAS_START)] 283 str r9, [r3, #4] 284 ldr r9, [r1, #(TD_MD + MD_RAS_END)] 285 str r9, [r3, #8] 286 287 /* Get the user structure for the new process in r9 */ 288 ldr r9, [r1, #(TD_PCB)] 289 290 /* r1 now free! */ 291 292 mrs r3, cpsr 293 /* 294 * We can do that, since 295 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 296 */ 297 orr r8, r3, #(PSR_UND32_MODE) 298 msr cpsr_c, r8 299 300 str sp, [r2, #(PCB_UND_SP)] 301 302 msr cpsr_c, r3 /* Restore the old mode */ 303 /* rem: r8 = old PCB */ 304 /* rem: r9 = new PCB */ 305 /* rem: interrupts are enabled */ 306 307 /* What else needs to be saved Only FPA stuff when that is supported */ 308 309 /* Third phase : restore saved context */ 310 311 /* rem: r8 = old PCB */ 312 /* rem: r9 = new PCB */ 313 /* rem: interrupts are enabled */ 314 315 ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ 316 mov r2, #DOMAIN_CLIENT 317 cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 318 beq .Lcs_context_switched /* Yup. Don't flush cache */ 319 mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ 320 /* 321 * Get the new L1 table pointer into r11. If we're switching to 322 * an LWP with the same address space as the outgoing one, we can 323 * skip the cache purge and the TTB load. 324 * 325 * To avoid data dep stalls that would happen anyway, we try 326 * and get some useful work done in the mean time. 327 */ 328 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 329 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 330 331 332 teq r10, r11 /* Same L1? */ 333 cmpeq r0, r5 /* Same DACR? */ 334 beq .Lcs_context_switched /* yes! */ 335 336 /* 337 * Definately need to flush the cache. 338 */ 339 340 ldr r1, .Lcpufuncs 341 mov lr, pc 342 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 343.Lcs_cache_purge_skipped: 344 /* rem: r6 = lock */ 345 /* rem: r9 = new PCB */ 346 /* rem: r10 = old L1 */ 347 /* rem: r11 = new L1 */ 348 349 mov r2, #0x00000000 350 ldr r7, [r9, #(PCB_PL1VEC)] 351 352 /* 353 * Ensure the vector table is accessible by fixing up the L1 354 */ 355 cmp r7, #0 /* No need to fixup vector table? */ 356 ldrne r2, [r7] /* But if yes, fetch current value */ 357 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 358 mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ 359 cmpne r2, r0 /* Stuffing the same value? */ 360#ifndef PMAP_INCLUDE_PTE_SYNC 361 strne r0, [r7] /* Nope, update it */ 362#else 363 beq .Lcs_same_vector 364 str r0, [r7] /* Otherwise, update it */ 365 366 /* 367 * Need to sync the cache to make sure that last store is 368 * visible to the MMU. 369 */ 370 ldr r2, .Lcpufuncs 371 mov r0, r7 372 mov r1, #4 373 mov lr, pc 374 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 375 376.Lcs_same_vector: 377#endif /* PMAP_INCLUDE_PTE_SYNC */ 378 379 cmp r10, r11 /* Switching to the same L1? */ 380 ldr r10, .Lcpufuncs 381 beq .Lcs_same_l1 /* Yup. */ 382 /* 383 * Do a full context switch, including full TLB flush. 384 */ 385 mov r0, r11 386 mov lr, pc 387 ldr pc, [r10, #CF_CONTEXT_SWITCH] 388 389 b .Lcs_context_switched 390 391 /* 392 * We're switching to a different process in the same L1. 393 * In this situation, we only need to flush the TLB for the 394 * vector_page mapping, and even then only if r7 is non-NULL. 395 */ 396.Lcs_same_l1: 397 cmp r7, #0 398 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 399 movne lr, pc 400 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 401 /* 402 * We can do that, since 403 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 404 */ 405 406.Lcs_context_switched: 407 408 /* Release the old thread */ 409 str r6, [r4, #TD_LOCK] 410 ldr r6, .Lblocked_lock 411 ldr r3, .Lcurthread 412 ldr r3, [r3] 413 4141: 415 ldr r4, [r3, #TD_LOCK] 416 cmp r4, r6 417 beq 1b 418 419 /* XXXSCW: Safe to re-enable FIQs here */ 420 421 /* rem: r9 = new PCB */ 422 423 mrs r3, cpsr 424 /* 425 * We can do that, since 426 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 427 */ 428 orr r2, r3, #(PSR_UND32_MODE) 429 msr cpsr_c, r2 430 431 ldr sp, [r9, #(PCB_UND_SP)] 432 433 msr cpsr_c, r3 /* Restore the old mode */ 434 /* Restore all the save registers */ 435#ifndef _ARM_ARCH_5E 436 add r7, r9, #PCB_R8 437 ldmia r7, {r8-r13} 438 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 439#else 440 mov r7, r9 441 ldr r8, [r7, #(PCB_R8)] 442 ldr r9, [r7, #(PCB_R9)] 443 ldr r10, [r7, #(PCB_R10)] 444 ldr r11, [r7, #(PCB_R11)] 445 ldr r12, [r7, #(PCB_R12)] 446 ldr r13, [r7, #(PCB_SP)] 447#endif 448 449 /* rem: r6 = lock */ 450 /* rem: r7 = new pcb */ 451 452#ifdef ARMFPE 453 add r0, r7, #(USER_SIZE) & 0x00ff 454 add r0, r0, #(USER_SIZE) & 0xff00 455 bl _C_LABEL(arm_fpe_core_changecontext) 456#endif 457 458 /* rem: r5 = new lwp's proc */ 459 /* rem: r6 = lock */ 460 /* rem: r7 = new PCB */ 461 462.Lswitch_return: 463 464 /* 465 * Pull the registers that got pushed when either savectx() or 466 * cpu_switch() was called and return. 467 */ 468 ldmfd sp!, {r4-r7, pc} 469#ifdef DIAGNOSTIC 470.Lswitch_bogons: 471 adr r0, .Lswitch_panic_str 472 bl _C_LABEL(panic) 4731: nop 474 b 1b 475 476.Lswitch_panic_str: 477 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" 478#endif 479ENTRY(savectx) 480 stmfd sp!, {r4-r7, lr} 481 /* 482 * r0 = pcb 483 */ 484 /* Store all the registers in the process's pcb */ 485 add r2, r0, #(PCB_R8) 486 stmia r2, {r8-r13} 487 ldmfd sp!, {r4-r7, pc} 488 489ENTRY(fork_trampoline) 490 mov r1, r5 491 mov r2, sp 492 mov r0, r4 493 mov fp, #0 494 bl _C_LABEL(fork_exit) 495 /* Kill irq"s */ 496 mrs r0, cpsr 497 orr r0, r0, #(I32_bit|F32_bit) 498 msr cpsr_c, r0 499 DO_AST 500 PULLFRAME 501 502 movs pc, lr /* Exit */ 503 504AST_LOCALS 505