1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * This file contains low-level cache management functions 4 * used for sleep and CPU speed changes on Apple machines. 5 * (In fact the only thing that is Apple-specific is that we assume 6 * that we can read from ROM at physical address 0xfff00000.) 7 * 8 * Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and 9 * Benjamin Herrenschmidt (benh@kernel.crashing.org) 10 */ 11 12#include <asm/processor.h> 13#include <asm/ppc_asm.h> 14#include <asm/cputable.h> 15#include <asm/feature-fixups.h> 16 17/* 18 * Flush and disable all data caches (dL1, L2, L3). This is used 19 * when going to sleep, when doing a PMU based cpufreq transition, 20 * or when "offlining" a CPU on SMP machines. This code is over 21 * paranoid, but I've had enough issues with various CPU revs and 22 * bugs that I decided it was worth being over cautious 23 */ 24 25_GLOBAL(flush_disable_caches) 26#ifndef CONFIG_PPC_BOOK3S_32 27 blr 28#else 29BEGIN_FTR_SECTION 30 b flush_disable_745x 31END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450) 32BEGIN_FTR_SECTION 33 b flush_disable_75x 34END_FTR_SECTION_IFSET(CPU_FTR_L2CR) 35 b __flush_disable_L1 36 37/* This is the code for G3 and 74[01]0 */ 38flush_disable_75x: 39 mflr r10 40 41 /* Turn off EE and DR in MSR */ 42 mfmsr r11 43 rlwinm r0,r11,0,~MSR_EE 44 rlwinm r0,r0,0,~MSR_DR 45 sync 46 mtmsr r0 47 isync 48 49 /* Stop DST streams */ 50BEGIN_FTR_SECTION 51 PPC_DSSALL 52 sync 53END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 54 55 /* Stop DPM */ 56 mfspr r8,SPRN_HID0 /* Save SPRN_HID0 in r8 */ 57 rlwinm r4,r8,0,12,10 /* Turn off HID0[DPM] */ 58 sync 59 mtspr SPRN_HID0,r4 /* Disable DPM */ 60 sync 61 62 /* Disp-flush L1. We have a weird problem here that I never 63 * totally figured out. On 750FX, using the ROM for the flush 64 * results in a non-working flush. We use that workaround for 65 * now until I finally understand what's going on. --BenH 66 */ 67 68 /* ROM base by default */ 69 lis r4,0xfff0 70 mfpvr r3 71 srwi r3,r3,16 72 cmplwi cr0,r3,0x7000 73 bne+ 1f 74 /* RAM base on 750FX */ 75 li r4,0 761: li r4,0x4000 77 mtctr r4 781: lwz r0,0(r4) 79 addi r4,r4,32 80 bdnz 1b 81 sync 82 isync 83 84 /* Disable / invalidate / enable L1 data */ 85 mfspr r3,SPRN_HID0 86 rlwinm r3,r3,0,~(HID0_DCE | HID0_ICE) 87 mtspr SPRN_HID0,r3 88 sync 89 isync 90 ori r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI) 91 sync 92 isync 93 mtspr SPRN_HID0,r3 94 xori r3,r3,(HID0_DCI|HID0_ICFI) 95 mtspr SPRN_HID0,r3 96 sync 97 98 /* Get the current enable bit of the L2CR into r4 */ 99 mfspr r5,SPRN_L2CR 100 /* Set to data-only (pre-745x bit) */ 101 oris r3,r5,L2CR_L2DO@h 102 b 2f 103 /* When disabling L2, code must be in L1 */ 104 .balign 32 1051: mtspr SPRN_L2CR,r3 1063: sync 107 isync 108 b 1f 1092: b 3f 1103: sync 111 isync 112 b 1b 1131: /* disp-flush L2. The interesting thing here is that the L2 can be 114 * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory 115 * but that is probbaly fine. We disp-flush over 4Mb to be safe 116 */ 117 lis r4,2 118 mtctr r4 119 lis r4,0xfff0 1201: lwz r0,0(r4) 121 addi r4,r4,32 122 bdnz 1b 123 sync 124 isync 125 lis r4,2 126 mtctr r4 127 lis r4,0xfff0 1281: dcbf 0,r4 129 addi r4,r4,32 130 bdnz 1b 131 sync 132 isync 133 134 /* now disable L2 */ 135 rlwinm r5,r5,0,~L2CR_L2E 136 b 2f 137 /* When disabling L2, code must be in L1 */ 138 .balign 32 1391: mtspr SPRN_L2CR,r5 1403: sync 141 isync 142 b 1f 1432: b 3f 1443: sync 145 isync 146 b 1b 1471: sync 148 isync 149 /* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */ 150 oris r4,r5,L2CR_L2I@h 151 mtspr SPRN_L2CR,r4 152 sync 153 isync 154 155 /* Wait for the invalidation to complete */ 1561: mfspr r3,SPRN_L2CR 157 rlwinm. r0,r3,0,31,31 158 bne 1b 159 160 /* Clear L2I */ 161 xoris r4,r4,L2CR_L2I@h 162 sync 163 mtspr SPRN_L2CR,r4 164 sync 165 166 /* now disable the L1 data cache */ 167 mfspr r0,SPRN_HID0 168 rlwinm r0,r0,0,~(HID0_DCE|HID0_ICE) 169 mtspr SPRN_HID0,r0 170 sync 171 isync 172 173 /* Restore HID0[DPM] to whatever it was before */ 174 sync 175 mfspr r0,SPRN_HID0 176 rlwimi r0,r8,0,11,11 /* Turn back HID0[DPM] */ 177 mtspr SPRN_HID0,r0 178 sync 179 180 /* restore DR and EE */ 181 sync 182 mtmsr r11 183 isync 184 185 mtlr r10 186 blr 187_ASM_NOKPROBE_SYMBOL(flush_disable_75x) 188 189/* This code is for 745x processors */ 190flush_disable_745x: 191 /* Turn off EE and DR in MSR */ 192 mfmsr r11 193 rlwinm r0,r11,0,~MSR_EE 194 rlwinm r0,r0,0,~MSR_DR 195 sync 196 mtmsr r0 197 isync 198 199 /* Stop prefetch streams */ 200 PPC_DSSALL 201 sync 202 203 /* Disable L2 prefetching */ 204 mfspr r0,SPRN_MSSCR0 205 rlwinm r0,r0,0,0,29 206 mtspr SPRN_MSSCR0,r0 207 sync 208 isync 209 lis r4,0 210 dcbf 0,r4 211 dcbf 0,r4 212 dcbf 0,r4 213 dcbf 0,r4 214 dcbf 0,r4 215 dcbf 0,r4 216 dcbf 0,r4 217 dcbf 0,r4 218 219 /* Due to a bug with the HW flush on some CPU revs, we occasionally 220 * experience data corruption. I'm adding a displacement flush along 221 * with a dcbf loop over a few Mb to "help". The problem isn't totally 222 * fixed by this in theory, but at least, in practice, I couldn't reproduce 223 * it even with a big hammer... 224 */ 225 226 lis r4,0x0002 227 mtctr r4 228 li r4,0 2291: 230 lwz r0,0(r4) 231 addi r4,r4,32 /* Go to start of next cache line */ 232 bdnz 1b 233 isync 234 235 /* Now, flush the first 4MB of memory */ 236 lis r4,0x0002 237 mtctr r4 238 li r4,0 239 sync 2401: 241 dcbf 0,r4 242 addi r4,r4,32 /* Go to start of next cache line */ 243 bdnz 1b 244 245 /* Flush and disable the L1 data cache */ 246 mfspr r6,SPRN_LDSTCR 247 lis r3,0xfff0 /* read from ROM for displacement flush */ 248 li r4,0xfe /* start with only way 0 unlocked */ 249 li r5,128 /* 128 lines in each way */ 2501: mtctr r5 251 rlwimi r6,r4,0,24,31 252 mtspr SPRN_LDSTCR,r6 253 sync 254 isync 2552: lwz r0,0(r3) /* touch each cache line */ 256 addi r3,r3,32 257 bdnz 2b 258 rlwinm r4,r4,1,24,30 /* move on to the next way */ 259 ori r4,r4,1 260 cmpwi r4,0xff /* all done? */ 261 bne 1b 262 /* now unlock the L1 data cache */ 263 li r4,0 264 rlwimi r6,r4,0,24,31 265 sync 266 mtspr SPRN_LDSTCR,r6 267 sync 268 isync 269 270 /* Flush the L2 cache using the hardware assist */ 271 mfspr r3,SPRN_L2CR 272 cmpwi r3,0 /* check if it is enabled first */ 273 bge 4f 274 oris r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h 275 b 2f 276 /* When disabling/locking L2, code must be in L1 */ 277 .balign 32 2781: mtspr SPRN_L2CR,r0 /* lock the L2 cache */ 2793: sync 280 isync 281 b 1f 2822: b 3f 2833: sync 284 isync 285 b 1b 2861: sync 287 isync 288 ori r0,r3,L2CR_L2HWF_745x 289 sync 290 mtspr SPRN_L2CR,r0 /* set the hardware flush bit */ 2913: mfspr r0,SPRN_L2CR /* wait for it to go to 0 */ 292 andi. r0,r0,L2CR_L2HWF_745x 293 bne 3b 294 sync 295 rlwinm r3,r3,0,~L2CR_L2E 296 b 2f 297 /* When disabling L2, code must be in L1 */ 298 .balign 32 2991: mtspr SPRN_L2CR,r3 /* disable the L2 cache */ 3003: sync 301 isync 302 b 1f 3032: b 3f 3043: sync 305 isync 306 b 1b 3071: sync 308 isync 309 oris r4,r3,L2CR_L2I@h 310 mtspr SPRN_L2CR,r4 311 sync 312 isync 3131: mfspr r4,SPRN_L2CR 314 andis. r0,r4,L2CR_L2I@h 315 bne 1b 316 sync 317 318BEGIN_FTR_SECTION 319 /* Flush the L3 cache using the hardware assist */ 3204: mfspr r3,SPRN_L3CR 321 cmpwi r3,0 /* check if it is enabled */ 322 bge 6f 323 oris r0,r3,L3CR_L3IO@h 324 ori r0,r0,L3CR_L3DO 325 sync 326 mtspr SPRN_L3CR,r0 /* lock the L3 cache */ 327 sync 328 isync 329 ori r0,r0,L3CR_L3HWF 330 sync 331 mtspr SPRN_L3CR,r0 /* set the hardware flush bit */ 3325: mfspr r0,SPRN_L3CR /* wait for it to go to zero */ 333 andi. r0,r0,L3CR_L3HWF 334 bne 5b 335 rlwinm r3,r3,0,~L3CR_L3E 336 sync 337 mtspr SPRN_L3CR,r3 /* disable the L3 cache */ 338 sync 339 ori r4,r3,L3CR_L3I 340 mtspr SPRN_L3CR,r4 3411: mfspr r4,SPRN_L3CR 342 andi. r0,r4,L3CR_L3I 343 bne 1b 344 sync 345END_FTR_SECTION_IFSET(CPU_FTR_L3CR) 346 3476: mfspr r0,SPRN_HID0 /* now disable the L1 data cache */ 348 rlwinm r0,r0,0,~HID0_DCE 349 mtspr SPRN_HID0,r0 350 sync 351 isync 352 mtmsr r11 /* restore DR and EE */ 353 isync 354 blr 355_ASM_NOKPROBE_SYMBOL(flush_disable_745x) 356#endif /* CONFIG_PPC_BOOK3S_32 */ 357