1 /* $NetBSD: cache.c,v 1.52 2016/07/11 23:06:54 matt Exp $ */
2
3 /*
4 * Copyright 2001, 2002 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 /*
39 * Copyright 2000, 2001
40 * Broadcom Corporation. All rights reserved.
41 *
42 * This software is furnished under license and may be used and copied only
43 * in accordance with the following terms and conditions. Subject to these
44 * conditions, you may download, copy, install, use, modify and distribute
45 * modified or unmodified copies of this software in source and/or binary
46 * form. No title or ownership is transferred hereby.
47 *
48 * 1) Any source code used, modified or distributed must reproduce and
49 * retain this copyright notice and list of conditions as they appear in
50 * the source file.
51 *
52 * 2) No right is granted to use any trade name, trademark, or logo of
53 * Broadcom Corporation. The "Broadcom Corporation" name may not be
54 * used to endorse or promote products derived from this software
55 * without the prior written permission of Broadcom Corporation.
56 *
57 * 3) THIS SOFTWARE IS PROVIDED "AS-IS" AND ANY EXPRESS OR IMPLIED
58 * WARRANTIES, INCLUDING BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES OF
59 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
60 * NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM BE LIABLE
61 * FOR ANY DAMAGES WHATSOEVER, AND IN PARTICULAR, BROADCOM SHALL NOT BE
62 * LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
63 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
64 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
65 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
66 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
67 * OR OTHERWISE), EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
68 */
69
70 #include <sys/cdefs.h>
71 __KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.52 2016/07/11 23:06:54 matt Exp $");
72
73 #include "opt_cputype.h"
74 #include "opt_mips_cache.h"
75
76 #include <sys/param.h>
77
78 #include <uvm/uvm_extern.h>
79
80 #include <mips/cache.h>
81 #include <mips/locore.h>
82
83 #ifdef MIPS1
84 #include <mips/cache_r3k.h>
85 #endif
86
87 #ifdef MIPS3_PLUS
88 #include <mips/cache_r4k.h>
89 #include <mips/cache_r5k.h>
90 #ifdef ENABLE_MIPS4_CACHE_R10K
91 #include <mips/cache_r10k.h>
92 #endif
93 #ifdef MIPS3_LOONGSON2
94 #include <mips/cache_ls2.h>
95 #endif
96 #endif
97
98 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
99 #include <mips/mipsNN.h> /* MIPS32/MIPS64 registers */
100 #include <mips/cache_mipsNN.h>
101 #ifdef MIPS64_OCTEON
102 #include <mips/cache_octeon.h>
103 #endif
104 #endif
105
106 #ifdef MIPS1
107 #ifdef ENABLE_MIPS_TX3900
108 #include <mips/cache_tx39.h>
109 void tx3900_get_cache_config(void);
110 void tx3920_get_cache_config(void);
111 void tx39_cache_config_write_through(void);
112 #endif /* ENABLE_MIPS_TX3900 */
113 #endif /* MIPS1 */
114
115 #if defined(MIPS3) || defined(MIPS4)
116 void mips3_get_cache_config(int);
117 #ifdef ENABLE_MIPS4_CACHE_R10K
118 void mips4_get_cache_config(int);
119 #endif /* ENABLE_MIPS4_CACHE_R10K */
120 #endif /* MIPS3 || MIPS4 */
121
122 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
123 static void mips_config_cache_prehistoric(void);
124 static void mips_config_cache_emips(void);
125 #endif
126 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
127 static void mips_config_cache_modern(uint32_t);
128 #endif
129
130 #if (MIPS1 + MIPS3 + MIPS4 + MIPS32 + MIPS64 + MIPS32R2 + MIPS64R2) > 0
131 /* no-cache definition */
132 static void no_cache_op(void);
133 static void no_cache_op_range(register_t va, vsize_t size);
134 static void no_cache_op_range_index(vaddr_t va, vsize_t size);
135
136 /* no-cache implementation */
no_cache_op(void)137 static void no_cache_op(void) {}
no_cache_op_range(register_t va,vsize_t size)138 static void no_cache_op_range(register_t va, vsize_t size) {}
no_cache_op_range_index(vaddr_t va,vsize_t size)139 static void no_cache_op_range_index(vaddr_t va, vsize_t size) {}
140 #endif
141
142 struct mips_cache_info mips_cache_info;
143 struct mips_cache_ops mips_cache_ops = {
144 .mco_intern_icache_sync_range_index = no_cache_op_range_index,
145 .mco_intern_icache_sync_range = no_cache_op_range,
146 .mco_intern_pdcache_sync_all= no_cache_op,
147 .mco_intern_pdcache_sync_range_index = no_cache_op_range_index,
148 .mco_intern_pdcache_sync_range = no_cache_op_range,
149 .mco_intern_pdcache_wbinv_range_index = no_cache_op_range_index,
150 .mco_intern_sdcache_sync_all= no_cache_op,
151 .mco_intern_sdcache_sync_range_index = no_cache_op_range_index,
152 .mco_intern_sdcache_sync_range = no_cache_op_range,
153 .mco_intern_sdcache_wbinv_range_index = no_cache_op_range_index,
154 };
155
156 /*
157 * mips_dcache_compute_align:
158 *
159 * Compute the D-cache alignment values.
160 */
161 void
mips_dcache_compute_align(void)162 mips_dcache_compute_align(void)
163 {
164 struct mips_cache_info * const mci = &mips_cache_info;
165 u_int align;
166
167 align = mci->mci_pdcache_line_size;
168
169 if (mci->mci_sdcache_line_size > align)
170 align = mci->mci_sdcache_line_size;
171
172 if (mci->mci_tcache_line_size > align)
173 align = mci->mci_tcache_line_size;
174
175 mci->mci_dcache_align = align;
176 mci->mci_dcache_align_mask = align - 1;
177 }
178
179 /*
180 * mips_config_cache:
181 *
182 * Configure the cache for the system.
183 *
184 * XXX DOES NOT HANDLE SPLIT SECONDARY CACHES.
185 */
186 void
mips_config_cache(void)187 mips_config_cache(void)
188 {
189 #ifdef DIAGNOSTIC
190 struct mips_cache_info * const mci = &mips_cache_info;
191 struct mips_cache_ops * const mco = &mips_cache_ops;
192 #endif
193 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
194
195 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
196 if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_PREHISTORIC)
197 mips_config_cache_prehistoric();
198 else if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_MICROSOFT)
199 mips_config_cache_emips();
200 #endif
201 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
202 if (MIPS_PRID_CID(cpu_id) != MIPS_PRID_CID_PREHISTORIC)
203 mips_config_cache_modern(cpu_id);
204 #endif
205
206 #ifdef DIAGNOSTIC
207 /* Check that all cache ops are set up. */
208 if (mci->mci_picache_size || 1) { /* XXX- must have primary Icache */
209 if (!mco->mco_icache_sync_all)
210 panic("no icache_sync_all cache op");
211 if (!mco->mco_icache_sync_range)
212 panic("no icache_sync_range cache op");
213 if (!mco->mco_icache_sync_range_index)
214 panic("no icache_sync_range_index cache op");
215 }
216 if (mci->mci_pdcache_size || 1) { /* XXX- must have primary Dcache */
217 if (!mco->mco_pdcache_wbinv_all)
218 panic("no pdcache_wbinv_all");
219 if (!mco->mco_pdcache_wbinv_range)
220 panic("no pdcache_wbinv_range");
221 if (!mco->mco_pdcache_wbinv_range_index)
222 panic("no pdcache_wbinv_range_index");
223 if (!mco->mco_pdcache_inv_range)
224 panic("no pdcache_inv_range");
225 if (!mco->mco_pdcache_wb_range)
226 panic("no pdcache_wb_range");
227 }
228 if (mci->mci_sdcache_size) {
229 if (!mco->mco_sdcache_wbinv_all)
230 panic("no sdcache_wbinv_all");
231 if (!mco->mco_sdcache_wbinv_range)
232 panic("no sdcache_wbinv_range");
233 if (!mco->mco_sdcache_wbinv_range_index)
234 panic("no sdcache_wbinv_range_index");
235 if (!mco->mco_sdcache_inv_range)
236 panic("no sdcache_inv_range");
237 if (!mco->mco_sdcache_wb_range)
238 panic("no sdcache_wb_range");
239 }
240 #endif /* DIAGNOSTIC */
241 }
242
243 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
244 /*
245 * XXX DOES NOT HANDLE SPLIT SECONDARY CACHES.
246 */
247 void
mips_config_cache_prehistoric(void)248 mips_config_cache_prehistoric(void)
249 {
250 struct mips_cache_info * const mci = &mips_cache_info;
251 struct mips_cache_ops * const mco = &mips_cache_ops;
252 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
253 #if defined(MIPS3) || defined(MIPS4)
254 int csizebase = MIPS3_CONFIG_C_DEFBASE;
255 #endif
256
257 KASSERT(PAGE_SIZE != 0);
258
259 /*
260 * Configure primary caches.
261 */
262 switch (MIPS_PRID_IMPL(cpu_id)) {
263 #ifdef MIPS1
264 case MIPS_R2000:
265 case MIPS_R3000:
266 mci->mci_picache_size = r3k_picache_size();
267 mci->mci_pdcache_size = r3k_pdcache_size();
268
269 mci->mci_picache_line_size = 4;
270 mci->mci_pdcache_line_size = 4;
271
272 mci->mci_picache_ways = 1;
273 mci->mci_pdcache_ways = 1;
274
275 mci->mci_pdcache_write_through = true;
276
277 mco->mco_icache_sync_all =
278 r3k_icache_sync_all;
279 mco->mco_icache_sync_range =
280 r3k_icache_sync_range;
281 mco->mco_icache_sync_range_index =
282 (void (*)(vaddr_t, vsize_t)) mco->mco_icache_sync_range;
283
284 mco->mco_pdcache_wbinv_all =
285 r3k_pdcache_wbinv_all;
286 mco->mco_pdcache_wbinv_range =
287 r3k_pdcache_inv_range;
288 mco->mco_pdcache_wbinv_range_index =
289 (void (*)(vaddr_t, vsize_t)) mco->mco_pdcache_wbinv_range;
290 mco->mco_pdcache_inv_range =
291 r3k_pdcache_inv_range;
292 mco->mco_pdcache_wb_range =
293 r3k_pdcache_wb_range;
294
295 uvmexp.ncolors = atop(mci->mci_pdcache_size);
296 break;
297
298 #ifdef ENABLE_MIPS_TX3900
299 case MIPS_TX3900:
300 switch (MIPS_PRID_REV_MAJ(cpu_id)) {
301 case 1: /* TX3912 */
302 mci->mci_picache_ways = 1;
303 mci->mci_picache_line_size = 16;
304 mci->mci_pdcache_line_size = 4;
305
306 tx3900_get_cache_config();
307
308 mci->mci_pdcache_write_through = true;
309
310 mco->mco_icache_sync_all =
311 tx3900_icache_sync_all_16;
312 mco->mco_icache_sync_range =
313 tx3900_icache_sync_range_16;
314 mco->mco_icache_sync_range_index =
315 (void (*)(vaddr_t, vsize_t))
316 tx3900_icache_sync_range_16;
317
318 mco->mco_pdcache_wbinv_all =
319 tx3900_pdcache_wbinv_all_4;
320 mco->mco_pdcache_wbinv_range =
321 tx3900_pdcache_inv_range_4;
322 mco->mco_pdcache_wbinv_range_index =
323 (void (*)(vaddr_t, vsize_t))
324 tx3900_pdcache_inv_range_4;
325 mco->mco_pdcache_inv_range =
326 tx3900_pdcache_inv_range_4;
327 mco->mco_pdcache_wb_range =
328 tx3900_pdcache_wb_range_4;
329 break;
330
331 case 3: /* TX3922 */
332 mci->mci_picache_ways = 2;
333 mci->mci_picache_line_size = 16;
334 mci->mci_pdcache_line_size = 16;
335
336 tx3920_get_cache_config();
337
338 mco->mco_icache_sync_all =
339 mci->mci_pdcache_write_through ?
340 tx3900_icache_sync_all_16 :
341 tx3920_icache_sync_all_16wb;
342 mco->mco_icache_sync_range =
343 mci->mci_pdcache_write_through ?
344 tx3920_icache_sync_range_16wt :
345 tx3920_icache_sync_range_16wb;
346 mco->mco_icache_sync_range_index =
347 (void (*)(vaddr_t, vsize_t))
348 mco->mco_icache_sync_range;
349
350 mco->mco_pdcache_wbinv_all =
351 mci->mci_pdcache_write_through ?
352 tx3920_pdcache_wbinv_all_16wt :
353 tx3920_pdcache_wbinv_all_16wb;
354 mco->mco_pdcache_wbinv_range =
355 mci->mci_pdcache_write_through ?
356 tx3920_pdcache_inv_range_16 :
357 tx3920_pdcache_wbinv_range_16wb;
358 mco->mco_pdcache_wbinv_range_index =
359 (void (*)(vaddr_t, vsize_t))
360 mco->mco_pdcache_wbinv_range;
361 mco->mco_pdcache_inv_range =
362 tx3920_pdcache_inv_range_16;
363 mco->mco_pdcache_wb_range =
364 mci->mci_pdcache_write_through ?
365 tx3920_pdcache_wb_range_16wt :
366 tx3920_pdcache_wb_range_16wb;
367 break;
368
369 default:
370 panic("mips_config_cache: unsupported TX3900");
371 }
372
373 mci->mci_pdcache_ways = 2;
374 tx3900_get_cache_config();
375 /* change to write-through mode */
376 tx39_cache_config_write_through();
377
378 uvmexp.ncolors = atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
379 break;
380 #endif /* ENABLE_MIPS_TX3900 */
381 #endif /* MIPS1 */
382
383 #if defined(MIPS3) || defined(MIPS4)
384 case MIPS_R4100:
385 if ((mips3_cp0_config_read() & MIPS3_CONFIG_CS) != 0)
386 csizebase = MIPS3_CONFIG_C_4100BASE;
387
388 /*
389 * R4100 (NEC VR series) revision number means:
390 *
391 * MIPS_PRID_REV_MAJ MIPS_PRID_REV_MIN
392 * VR4102 4 ?
393 * VR4111 5 ?
394 * VR4181 5 ?
395 * VR4121 6 ?
396 * VR4122 7 0 or 1
397 * VR4181A 7 3 <
398 * VR4131 8 ?
399 */
400 /* Vr4131 has R4600 style 2-way set-associative cache */
401 if (MIPS_PRID_REV_MAJ(cpu_id) == 8)
402 goto primary_cache_is_2way;
403 /* FALLTHROUGH */
404
405 case MIPS_R4000:
406 case MIPS_R4300:
407 mci->mci_picache_ways = 1;
408 mci->mci_pdcache_ways = 1;
409 mci->mci_sdcache_ways = 1;
410
411 mips3_get_cache_config(csizebase);
412
413 /* no VCE support if there is no L2 cache */
414 if (mci->mci_picache_size > PAGE_SIZE)
415 mci->mci_icache_virtual_alias = true;
416 if (mci->mci_pdcache_size > PAGE_SIZE)
417 mci->mci_cache_virtual_alias = true;
418
419 mco->mco_icache_sync_all = r4k_icache_sync_all_generic;
420 switch (mci->mci_picache_line_size) {
421 case 16:
422 mco->mco_icache_sync_range =
423 cache_r4k_icache_hit_inv_16;
424 mco->mco_icache_sync_range_index =
425 cache_r4k_icache_index_inv_16;
426 break;
427
428 case 32:
429 mco->mco_icache_sync_range =
430 cache_r4k_icache_hit_inv_32;
431 mco->mco_icache_sync_range_index =
432 cache_r4k_icache_index_inv_32;
433 break;
434
435 default:
436 panic("r4k picache line size %d",
437 mci->mci_picache_line_size);
438 }
439
440 mco->mco_pdcache_wbinv_all = r4k_pdcache_wbinv_all_generic;
441 switch (mci->mci_pdcache_line_size) {
442 case 16:
443 mco->mco_pdcache_wbinv_range =
444 cache_r4k_pdcache_hit_wb_inv_16;
445 mco->mco_pdcache_wbinv_range_index =
446 cache_r4k_pdcache_index_wb_inv_16;
447 mco->mco_pdcache_inv_range =
448 cache_r4k_pdcache_hit_inv_16;
449 mco->mco_pdcache_wb_range =
450 cache_r4k_pdcache_hit_wb_16;
451 break;
452
453 case 32:
454 mco->mco_pdcache_wbinv_range =
455 cache_r4k_pdcache_hit_wb_inv_32;
456 mco->mco_pdcache_wbinv_range_index =
457 cache_r4k_pdcache_index_wb_inv_32;
458 mco->mco_pdcache_inv_range =
459 cache_r4k_pdcache_hit_inv_32;
460 mco->mco_pdcache_wb_range =
461 cache_r4k_pdcache_hit_wb_32;
462 break;
463
464 default:
465 panic("r4k pdcache line size %d",
466 mci->mci_pdcache_line_size);
467 }
468
469 /* Virtually-indexed cache; no use for colors. */
470 break;
471
472 case MIPS_R4600:
473 #ifdef ENABLE_MIPS_R4700
474 case MIPS_R4700:
475 #endif
476 #ifndef ENABLE_MIPS_R3NKK
477 case MIPS_R5000:
478 #endif
479 case MIPS_RM5200:
480 primary_cache_is_2way:
481 mci->mci_picache_ways = 2;
482 mci->mci_pdcache_ways = 2;
483
484 mips3_get_cache_config(csizebase);
485
486 if (mci->mci_picache_size / mci->mci_picache_ways > PAGE_SIZE)
487 mci->mci_icache_virtual_alias = true;
488 if (mci->mci_pdcache_size / mci->mci_pdcache_ways > PAGE_SIZE)
489 mci->mci_cache_virtual_alias = true;
490
491 mco->mco_icache_sync_all = r5k_picache_sync_all;
492 mco->mco_icache_sync_range = r5k_picache_sync_range;
493 mco->mco_icache_sync_range_index = r5k_picache_sync_range_index;
494
495 switch (mci->mci_picache_line_size) {
496 case 32:
497 /* used internally by mipsNN_picache_sync_range */
498 mco->mco_intern_icache_sync_range =
499 cache_r4k_icache_hit_inv_16;
500
501 /* used internally by mipsNN_picache_sync_range_index */
502 mco->mco_intern_icache_sync_range_index =
503 cache_r4k_icache_index_inv_16;
504 break;
505
506 default:
507 panic("r5k picache line size %u",
508 mci->mci_picache_line_size);
509 }
510
511 mco->mco_pdcache_wbinv_all = r5k_pdcache_wbinv_all;
512 mco->mco_pdcache_wbinv_range_index =
513 r5k_pdcache_wbinv_range_index;
514
515 switch (mci->mci_pdcache_line_size) {
516 case 16:
517 mco->mco_pdcache_wbinv_range =
518 cache_r4k_pdcache_hit_wb_inv_16;
519 mco->mco_pdcache_inv_range =
520 cache_r4k_pdcache_hit_inv_16;
521 mco->mco_pdcache_wb_range =
522 cache_r4k_pdcache_hit_wb_16;
523
524 /* used internally by r5k_pdcache_wbinv_range_index */
525 mco->mco_intern_pdcache_wbinv_range_index =
526 cache_r4k_pdcache_index_wb_inv_16;
527 break;
528
529 case 32:
530 mco->mco_pdcache_wbinv_range =
531 cache_r4k_pdcache_hit_wb_inv_32;
532 mco->mco_pdcache_inv_range =
533 cache_r4k_pdcache_hit_inv_32;
534 mco->mco_pdcache_wb_range =
535 cache_r4k_pdcache_hit_wb_32;
536
537 /* used internally by r5k_pdcache_wbinv_range_index */
538 mco->mco_intern_pdcache_wbinv_range_index =
539 cache_r4k_pdcache_index_wb_inv_32;
540 break;
541
542 default:
543 panic("r5k pdcache line size %d",
544 mci->mci_pdcache_line_size);
545 }
546
547 mco->mco_intern_pdcache_sync_all = mco->mco_pdcache_wbinv_all;
548 mco->mco_intern_pdcache_sync_range_index =
549 mco->mco_intern_pdcache_wbinv_range_index;
550 mco->mco_intern_pdcache_sync_range = mco->mco_pdcache_wb_range;
551
552 /*
553 * Deal with R4600 chip bugs.
554 */
555 if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4600 &&
556 MIPS_PRID_REV_MAJ(cpu_id) == 1) {
557 KASSERT(mci->mci_pdcache_line_size == 32);
558 mco->mco_pdcache_wbinv_range =
559 r4600v1_pdcache_wbinv_range_32;
560 mco->mco_pdcache_inv_range =
561 r4600v1_pdcache_inv_range_32;
562 mco->mco_pdcache_wb_range =
563 r4600v1_pdcache_wb_range_32;
564 } else if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4600 &&
565 MIPS_PRID_REV_MAJ(cpu_id) == 2) {
566 KASSERT(mci->mci_pdcache_line_size == 32);
567 mco->mco_pdcache_wbinv_range =
568 r4600v2_pdcache_wbinv_range_32;
569 mco->mco_pdcache_inv_range =
570 r4600v2_pdcache_inv_range_32;
571 mco->mco_pdcache_wb_range =
572 r4600v2_pdcache_wb_range_32;
573 }
574
575 /*
576 * Deal with VR4131 chip bugs.
577 */
578 if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4100 &&
579 MIPS_PRID_REV_MAJ(cpu_id) == 8) {
580 KASSERT(mci->mci_pdcache_line_size == 16);
581 mco->mco_pdcache_wbinv_range =
582 vr4131v1_pdcache_wbinv_range_16;
583 }
584
585 /* Virtually-indexed cache; no use for colors. */
586 break;
587 #ifdef ENABLE_MIPS4_CACHE_R10K
588 case MIPS_R10000:
589 case MIPS_R12000:
590 case MIPS_R14000:
591 mci->mci_picache_ways = 2;
592 mci->mci_pdcache_ways = 2;
593 mci->mci_sdcache_ways = 2;
594
595 mips4_get_cache_config(csizebase);
596
597 /* VCE is handled by hardware */
598
599 mco->mco_icache_sync_all =
600 r10k_icache_sync_all;
601 mco->mco_icache_sync_range =
602 r10k_icache_sync_range;
603 mco->mco_icache_sync_range_index =
604 r10k_icache_sync_range_index;
605 mco->mco_pdcache_wbinv_all =
606 r10k_pdcache_wbinv_all;
607 mco->mco_pdcache_wbinv_range =
608 r10k_pdcache_wbinv_range;
609 mco->mco_pdcache_wbinv_range_index =
610 r10k_pdcache_wbinv_range_index;
611 mco->mco_pdcache_inv_range =
612 r10k_pdcache_inv_range;
613 mco->mco_pdcache_wb_range =
614 r10k_pdcache_wb_range;
615 break;
616 #endif /* ENABLE_MIPS4_CACHE_R10K */
617 #ifdef MIPS3_LOONGSON2
618 case MIPS_LOONGSON2:
619 mci->mci_picache_ways = 4;
620 mci->mci_pdcache_ways = 4;
621
622 mips3_get_cache_config(csizebase);
623
624 mci->mci_sdcache_line_size = 32; /* don't trust config reg */
625
626 if (mci->mci_picache_size / mci->mci_picache_ways > PAGE_SIZE)
627 mci->mci_icache_virtual_alias = true;
628 if (mci->mci_pdcache_size / mci->mci_pdcache_ways > PAGE_SIZE)
629 mci->mci_cache_virtual_alias = true;
630
631 mco->mco_icache_sync_all =
632 ls2_icache_sync_all;
633 mco->mco_icache_sync_range =
634 ls2_icache_sync_range;
635 mco->mco_icache_sync_range_index =
636 ls2_icache_sync_range_index;
637
638 mco->mco_pdcache_wbinv_all =
639 ls2_pdcache_wbinv_all;
640 mco->mco_pdcache_wbinv_range =
641 ls2_pdcache_wbinv_range;
642 mco->mco_pdcache_wbinv_range_index =
643 ls2_pdcache_wbinv_range_index;
644 mco->mco_pdcache_inv_range =
645 ls2_pdcache_inv_range;
646 mco->mco_pdcache_wb_range =
647 ls2_pdcache_wb_range;
648
649 /*
650 * For current version chips, [the] operating system is
651 * obliged to eliminate the potential for virtual aliasing.
652 */
653 uvmexp.ncolors = mci->mci_pdcache_ways;
654 break;
655 #endif
656 #endif /* MIPS3 || MIPS4 */
657 default:
658 panic("can't handle primary cache on impl 0x%x",
659 MIPS_PRID_IMPL(cpu_id));
660 }
661
662 /*
663 * Compute the "way mask" for each cache.
664 */
665 if (mci->mci_picache_size) {
666 KASSERT(mci->mci_picache_ways != 0);
667 mci->mci_picache_way_size = mci->mci_picache_size / mci->mci_picache_ways;
668 mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
669 #if (MIPS2 + MIPS3) > 0
670 if (mci->mci_icache_virtual_alias)
671 mci->mci_icache_alias_mask =
672 mci->mci_picache_way_mask & -PAGE_SIZE;
673 #endif
674 }
675 if (mci->mci_pdcache_size) {
676 KASSERT(mci->mci_pdcache_ways != 0);
677 mci->mci_pdcache_way_size = mci->mci_pdcache_size / mci->mci_pdcache_ways;
678 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
679 #if (MIPS2 + MIPS3) > 0
680 if (mci->mci_cache_virtual_alias)
681 mci->mci_cache_alias_mask =
682 mci->mci_picache_way_mask & -PAGE_SIZE;
683 #endif
684 }
685
686 mips_dcache_compute_align();
687
688 if (mci->mci_sdcache_line_size == 0)
689 return;
690
691 /*
692 * Configure the secondary cache.
693 */
694 switch (MIPS_PRID_IMPL(cpu_id)) {
695 #if defined(MIPS3) || defined(MIPS4)
696 case MIPS_R4000:
697 /*
698 * R4000/R4400 detects virtual alias by VCE as if
699 * its primary cache size were 32KB, because it always
700 * compares 3 bits of vaddr[14:12] which causes
701 * primary cache miss and PIdx[2:0] in the secondary
702 * cache tag regardless of its primary cache size.
703 * i.e. VCE could happen even if there is no actual
704 * virtual alias on its 8KB or 16KB primary cache
705 * which has only 1 or 2 bit valid PIdx in 4KB page.
706 * Actual primary cache size is ignored wrt VCE
707 * and virtual aliases are resolved by the VCE hander,
708 * but it's still worth to avoid unnecessary VCE by
709 * setting alias mask and prefer mask to 32K, though
710 * some other possible aliases (maybe caused by KSEG0
711 * accesses which can't be managed by PMAP_PREFER(9))
712 * will still be resolved by the VCED/VCEI handler.
713 */
714 mci->mci_cache_alias_mask =
715 (MIPS3_MAX_PCACHE_SIZE - 1) & ~PAGE_MASK; /* va[14:12] */
716 mci->mci_cache_prefer_mask = MIPS3_MAX_PCACHE_SIZE - 1;
717
718 mci->mci_icache_virtual_alias = false;
719 mci->mci_cache_virtual_alias = false;
720 /* FALLTHROUGH */
721 case MIPS_R4600:
722 #ifdef ENABLE_MIPS_R4700
723 case MIPS_R4700:
724 #endif
725 switch (mci->mci_sdcache_ways) {
726 case 1:
727 mco->mco_sdcache_wbinv_all =
728 r4k_sdcache_wbinv_all_generic;
729 switch (mci->mci_sdcache_line_size) {
730 case 16:
731 mco->mco_sdcache_wbinv_range =
732 cache_r4k_sdcache_hit_wb_inv_16;
733 mco->mco_sdcache_wbinv_range_index =
734 cache_r4k_sdcache_index_wb_inv_16;
735 mco->mco_sdcache_inv_range =
736 cache_r4k_sdcache_hit_inv_16;
737 mco->mco_sdcache_wb_range =
738 cache_r4k_sdcache_hit_wb_16;
739 break;
740
741 case 32:
742 mco->mco_sdcache_wbinv_range =
743 cache_r4k_sdcache_hit_wb_inv_32;
744 mco->mco_sdcache_wbinv_range_index =
745 cache_r4k_sdcache_index_wb_inv_32;
746 mco->mco_sdcache_inv_range =
747 cache_r4k_sdcache_hit_inv_32;
748 mco->mco_sdcache_wb_range =
749 cache_r4k_sdcache_hit_wb_32;
750 break;
751
752 case 64:
753 mco->mco_sdcache_wbinv_range =
754 cache_r4k_sdcache_hit_wb_inv_64;
755 mco->mco_sdcache_wbinv_range_index =
756 cache_r4k_sdcache_index_wb_inv_64;
757 mco->mco_sdcache_inv_range =
758 cache_r4k_sdcache_hit_inv_64;
759 mco->mco_sdcache_wb_range =
760 cache_r4k_sdcache_hit_wb_64;
761 break;
762
763 case 128:
764 mco->mco_sdcache_wbinv_range =
765 cache_r4k_sdcache_hit_wb_inv_128;
766 mco->mco_sdcache_wbinv_range_index =
767 cache_r4k_sdcache_index_wb_inv_128;
768 mco->mco_sdcache_inv_range =
769 cache_r4k_sdcache_hit_inv_128;
770 mco->mco_sdcache_wb_range =
771 cache_r4k_sdcache_hit_wb_128;
772 break;
773
774 default:
775 panic("r4k sdcache %d way line size %d",
776 mci->mci_sdcache_ways, mci->mci_sdcache_line_size);
777 }
778 break;
779
780 default:
781 panic("r4k sdcache %d way line size %d",
782 mci->mci_sdcache_ways, mci->mci_sdcache_line_size);
783 }
784 break;
785 #ifndef ENABLE_MIPS_R3NKK
786 case MIPS_R5000:
787 #endif
788 case MIPS_RM5200:
789 mci->mci_sdcache_write_through = true;
790 mco->mco_sdcache_wbinv_all =
791 r5k_sdcache_wbinv_all;
792 mco->mco_sdcache_wbinv_range =
793 r5k_sdcache_wbinv_range;
794 mco->mco_sdcache_wbinv_range_index =
795 r5k_sdcache_wbinv_range_index;
796 mco->mco_sdcache_inv_range =
797 r5k_sdcache_wbinv_range;
798 mco->mco_sdcache_wb_range = no_cache_op_range;
799 break;
800 #ifdef ENABLE_MIPS4_CACHE_R10K
801 case MIPS_R10000:
802 case MIPS_R12000:
803 case MIPS_R14000:
804 mco->mco_sdcache_wbinv_all =
805 r10k_sdcache_wbinv_all;
806 mco->mco_sdcache_wbinv_range =
807 r10k_sdcache_wbinv_range;
808 mco->mco_sdcache_wbinv_range_index =
809 r10k_sdcache_wbinv_range_index;
810 mco->mco_sdcache_inv_range =
811 r10k_sdcache_inv_range;
812 mco->mco_sdcache_wb_range =
813 r10k_sdcache_wb_range;
814 break;
815 #endif /* ENABLE_MIPS4_CACHE_R10K */
816 #ifdef MIPS3_LOONGSON2
817 case MIPS_LOONGSON2:
818 mci->mci_sdcache_ways = 4;
819 mci->mci_sdcache_size = 512*1024;
820 mci->mci_scache_unified = 1;
821
822 mco->mco_sdcache_wbinv_all =
823 ls2_sdcache_wbinv_all;
824 mco->mco_sdcache_wbinv_range =
825 ls2_sdcache_wbinv_range;
826 mco->mco_sdcache_wbinv_range_index =
827 ls2_sdcache_wbinv_range_index;
828 mco->mco_sdcache_inv_range =
829 ls2_sdcache_inv_range;
830 mco->mco_sdcache_wb_range =
831 ls2_sdcache_wb_range;
832
833 /*
834 * The secondary cache is physically indexed and tagged
835 */
836 break;
837 #endif
838 #endif /* MIPS3 || MIPS4 */
839
840 default:
841 panic("can't handle secondary cache on impl 0x%x",
842 MIPS_PRID_IMPL(cpu_id));
843 }
844
845 /*
846 * Compute the "way mask" for each secondary cache.
847 */
848 if (mci->mci_sdcache_size) {
849 KASSERT(mci->mci_sdcache_ways != 0);
850 mci->mci_sdcache_way_size = mci->mci_sdcache_size / mci->mci_sdcache_ways;
851 mci->mci_sdcache_way_mask = mci->mci_sdcache_way_size - 1;
852 }
853
854 mips_dcache_compute_align();
855 }
856
857 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
858 void
mips_config_cache_emips(void)859 mips_config_cache_emips(void)
860 {
861 struct mips_cache_info * const mci = &mips_cache_info;
862 struct mips_cache_ops * const mco = &mips_cache_ops;
863 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
864 KASSERT(PAGE_SIZE != 0);
865
866 /*
867 * Configure primary caches.
868 */
869 switch (MIPS_PRID_IMPL(cpu_id)) {
870 case MIPS_eMIPS:
871 mci->mci_picache_size = 0;
872 mci->mci_pdcache_size = 0;
873
874 mci->mci_picache_line_size = 4;
875 mci->mci_pdcache_line_size = 4;
876
877 mci->mci_picache_ways = 1;
878 mci->mci_pdcache_ways = 1;
879
880 mci->mci_pdcache_write_through = true;
881
882 mco->mco_icache_sync_all = no_cache_op;
883 mco->mco_icache_sync_range = no_cache_op_range;
884 mco->mco_icache_sync_range_index = no_cache_op_range_index;
885
886 mco->mco_pdcache_wbinv_all = no_cache_op;
887 mco->mco_pdcache_wbinv_range = no_cache_op_range;
888 mco->mco_pdcache_wbinv_range_index = no_cache_op_range_index;
889 mco->mco_pdcache_inv_range = no_cache_op_range;
890 mco->mco_pdcache_wb_range = no_cache_op_range;
891
892 uvmexp.ncolors = 1;
893 break;
894
895 default:
896 panic("%s: unsupported eMIPS", __func__);
897 }
898 }
899 #endif
900
901 #ifdef MIPS1
902 #ifdef ENABLE_MIPS_TX3900
903 /*
904 * tx3900_get_cache_config:
905 *
906 * Fetch cache size information for the TX3900.
907 */
908 void
tx3900_get_cache_config(void)909 tx3900_get_cache_config(void)
910 {
911 struct mips_cache_info * const mci = &mips_cache_info;
912 uint32_t config;
913
914 config = tx3900_cp0_config_read();
915
916 mci->mci_picache_size = R3900_C_SIZE_MIN <<
917 ((config & R3900_CONFIG_ICS_MASK) >> R3900_CONFIG_ICS_SHIFT);
918
919 mci->mci_pdcache_size = R3900_C_SIZE_MIN <<
920 ((config & R3900_CONFIG_DCS_MASK) >> R3900_CONFIG_DCS_SHIFT);
921 }
922
923 /*
924 * tx3920_get_cache_config:
925 *
926 * Fetch cache size information for the TX3920.
927 */
928 void
tx3920_get_cache_config(void)929 tx3920_get_cache_config(void)
930 {
931 struct mips_cache_info * const mci = &mips_cache_info;
932
933 /* Size is the same as TX3900. */
934 tx3900_get_cache_config();
935
936 /* Now determine write-through/write-back mode. */
937 if ((tx3900_cp0_config_read() & R3900_CONFIG_WBON) == 0)
938 mci->mci_pdcache_write_through = true;
939 }
940
941 /*
942 * tx39_cache_config_write_through:
943 *
944 * TX3922 write-through D-cache mode.
945 * for TX3912, no meaning. (no write-back mode)
946 */
947 void
tx39_cache_config_write_through(void)948 tx39_cache_config_write_through(void)
949 {
950 u_int32_t r;
951
952 mips_dcache_wbinv_all();
953
954 __asm volatile("mfc0 %0, $3" : "=r"(r));
955 r &= 0xffffdfff;
956 __asm volatile("mtc0 %0, $3" : : "r"(r));
957 }
958
959 #endif /* ENABLE_MIPS_TX3900 */
960 #endif /* MIPS1 */
961
962 #if defined(MIPS3) || defined(MIPS4)
963 /*
964 * mips3_get_cache_config:
965 *
966 * Fetch the cache config information for a MIPS-3 or MIPS-4
967 * processor (virtually-indexed cache).
968 *
969 * NOTE: Fetching the size of the secondary cache is something
970 * that platform specific code has to do. We'd appreciate it
971 * if they initialized the size before now.
972 *
973 * ALSO NOTE: The number of ways in the cache must already be
974 * initialized.
975 */
976 void
mips3_get_cache_config(int csizebase)977 mips3_get_cache_config(int csizebase)
978 {
979 struct mips_cache_info * const mci = &mips_cache_info;
980 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
981 bool has_sdcache_enable = false;
982 uint32_t config = mips3_cp0_config_read();
983
984 mci->mci_picache_size = MIPS3_CONFIG_CACHE_SIZE(config,
985 MIPS3_CONFIG_IC_MASK, csizebase, MIPS3_CONFIG_IC_SHIFT);
986 mci->mci_picache_line_size = MIPS3_CONFIG_CACHE_L1_LSIZE(config,
987 MIPS3_CONFIG_IB);
988
989 mci->mci_pdcache_size = MIPS3_CONFIG_CACHE_SIZE(config,
990 MIPS3_CONFIG_DC_MASK, csizebase, MIPS3_CONFIG_DC_SHIFT);
991 mci->mci_pdcache_line_size = MIPS3_CONFIG_CACHE_L1_LSIZE(config,
992 MIPS3_CONFIG_DB);
993
994 mci->mci_icache_alias_mask =
995 (mci->mci_picache_size / mci->mci_picache_ways - 1) & -PAGE_SIZE;
996 mci->mci_cache_alias_mask =
997 (mci->mci_pdcache_size / mci->mci_pdcache_ways - 1) & -PAGE_SIZE;
998 mci->mci_cache_prefer_mask =
999 max(mci->mci_pdcache_size, mci->mci_picache_size) - 1;
1000 uvmexp.ncolors = (mci->mci_cache_alias_mask >> PAGE_SHIFT) + 1;
1001
1002 switch(MIPS_PRID_IMPL(cpu_id)) {
1003 #ifndef ENABLE_MIPS_R3NKK
1004 case MIPS_R5000:
1005 #endif
1006 case MIPS_RM5200:
1007 has_sdcache_enable = true;
1008 break;
1009 }
1010
1011 /*
1012 * If CPU has a software-enabled L2 cache, check both if it's
1013 * present and if it's enabled before making assumptions the
1014 * L2 is usable. If the L2 is disabled, we treat it the same
1015 * as if there were no L2 cache.
1016 */
1017 if ((config & MIPS3_CONFIG_SC) == 0) {
1018 if (has_sdcache_enable == 0 ||
1019 (has_sdcache_enable && (config & MIPS3_CONFIG_SE))) {
1020 mci->mci_sdcache_line_size =
1021 MIPS3_CONFIG_CACHE_L2_LSIZE(config);
1022 if ((config & MIPS3_CONFIG_SS) == 0)
1023 mci->mci_scache_unified = true;
1024 } else {
1025 #ifdef CACHE_DEBUG
1026 printf("External cache detected, but is disabled -- WILL NOT ENABLE!\n");
1027 #endif /* CACHE_DEBUG */
1028 }
1029 }
1030 }
1031
1032 #ifdef ENABLE_MIPS4_CACHE_R10K
1033 void
mips4_get_cache_config(int csizebase)1034 mips4_get_cache_config(int csizebase)
1035 {
1036 struct mips_cache_info * const mci = &mips_cache_info;
1037 uint32_t config = mips3_cp0_config_read();
1038
1039 mci->mci_picache_size = MIPS4_CONFIG_CACHE_SIZE(config,
1040 MIPS4_CONFIG_IC_MASK, csizebase, MIPS4_CONFIG_IC_SHIFT);
1041 mci->mci_picache_line_size = 64; /* 64 Byte */
1042
1043 mci->mci_pdcache_size = MIPS4_CONFIG_CACHE_SIZE(config,
1044 MIPS4_CONFIG_DC_MASK, csizebase, MIPS4_CONFIG_DC_SHIFT);
1045 mci->mci_pdcache_line_size = 32; /* 32 Byte */
1046
1047 mci->mci_cache_alias_mask =
1048 ((mci->mci_pdcache_size / mci->mci_pdcache_ways) - 1) & ~PAGE_MASK;
1049 mci->mci_cache_prefer_mask =
1050 max(mci->mci_pdcache_size, mci->mci_picache_size) - 1;
1051 }
1052 #endif /* ENABLE_MIPS4_CACHE_R10K */
1053 #endif /* MIPS3 || MIPS4 */
1054 #endif /* MIPS1 || MIPS3 || MIPS4 */
1055
1056 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
1057
1058 static void
mips_config_cache_modern(uint32_t cpu_id)1059 mips_config_cache_modern(uint32_t cpu_id)
1060 {
1061 struct mips_cache_info * const mci = &mips_cache_info;
1062 struct mips_cache_ops * const mco = &mips_cache_ops;
1063 struct mips_options * const opts = &mips_options;
1064 /* MIPS32/MIPS64, use coprocessor 0 config registers */
1065 uint32_t cfg, cfg1;
1066
1067 cfg = mips3_cp0_config_read();
1068 cfg1 = mipsNN_cp0_config1_read();
1069
1070 #ifdef MIPS_DISABLE_L1_CACHE
1071 cfg1 &= ~MIPSNN_CFG1_IL_MASK;
1072 cfg1 &= ~MIPSNN_CFG1_DL_MASK;
1073 mipsNN_cp0_config1_write(cfg1);
1074 #endif
1075
1076 /* figure out Dcache params. */
1077 switch (MIPSNN_GET(CFG1_DL, cfg1)) {
1078 case MIPSNN_CFG1_DL_NONE:
1079 #ifdef MIPS64_OCTEON
1080 mci->mci_pdcache_line_size = 128;
1081 mci->mci_pdcache_way_size = 256;
1082 mci->mci_pdcache_ways = 64;
1083 mci->mci_pdcache_write_through = true;
1084
1085 mci->mci_pdcache_size =
1086 mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
1087 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
1088 uvmexp.ncolors = atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
1089 #else
1090 mci->mci_pdcache_line_size = mci->mci_pdcache_way_size =
1091 mci->mci_pdcache_ways = 0;
1092 #endif
1093 break;
1094 case MIPSNN_CFG1_DL_RSVD:
1095 panic("reserved MIPS32/64 Dcache line size");
1096 break;
1097 default:
1098 if (MIPSNN_GET(CFG1_DS, cfg1) == MIPSNN_CFG1_DS_RSVD)
1099 panic("reserved MIPS32/64 Dcache sets per way");
1100 mci->mci_pdcache_line_size = MIPSNN_CFG1_DL(cfg1);
1101 mci->mci_pdcache_way_size =
1102 mci->mci_pdcache_line_size * MIPSNN_CFG1_DS(cfg1);
1103 mci->mci_pdcache_ways = MIPSNN_CFG1_DA(cfg1) + 1;
1104
1105 /*
1106 * Compute the total size and "way mask" for the
1107 * primary Dcache.
1108 */
1109 mci->mci_pdcache_size =
1110 mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
1111 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
1112 uvmexp.ncolors = atop(mci->mci_pdcache_way_size);
1113 break;
1114 }
1115
1116 /* figure out Icache params. */
1117 switch (MIPSNN_GET(CFG1_IL, cfg1)) {
1118 case MIPSNN_CFG1_IL_NONE:
1119 mci->mci_picache_line_size = mci->mci_picache_way_size =
1120 mci->mci_picache_ways = 0;
1121 break;
1122 case MIPSNN_CFG1_IL_RSVD:
1123 panic("reserved MIPS32/64 Icache line size");
1124 break;
1125 default:
1126 if (MIPSNN_GET(CFG1_IS, cfg1) == MIPSNN_CFG1_IS_RSVD)
1127 panic("reserved MIPS32/64 Icache sets per way");
1128 mci->mci_picache_line_size = MIPSNN_CFG1_IL(cfg1);
1129 mci->mci_picache_way_size =
1130 mci->mci_picache_line_size * MIPSNN_CFG1_IS(cfg1);
1131 mci->mci_picache_ways = MIPSNN_CFG1_IA(cfg1) + 1;
1132
1133 /*
1134 * Is this Icache virtually indexed and virtually tagged?
1135 */
1136 mci->mci_picache_vivt = (cfg & MIPSNN_CFG_VI) != 0;
1137
1138 /*
1139 * Compute the total size and "way mask" for the
1140 * primary Icache.
1141 */
1142 mci->mci_picache_size =
1143 mci->mci_picache_way_size * mci->mci_picache_ways;
1144 mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
1145 break;
1146 }
1147
1148 #define CACHE_DEBUG
1149 #ifdef CACHE_DEBUG
1150 printf("MIPS32/64 params: cpu arch: %d\n", opts->mips_cpu_arch);
1151 printf("MIPS32/64 params: TLB entries: %d\n", opts->mips_num_tlb_entries);
1152 if (mci->mci_picache_line_size == 0) {
1153 printf("MIPS32/64 params: no Icache\n");
1154 } else {
1155 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1156 "ways=%d, sets=%d, colors=%d\n", "Icache",
1157 mci->mci_picache_line_size,
1158 mci->mci_picache_way_size * mci->mci_picache_ways,
1159 mci->mci_picache_ways,
1160 mci->mci_picache_way_size / mci->mci_picache_line_size,
1161 mci->mci_picache_way_size >> PAGE_SHIFT);
1162 }
1163 if (mci->mci_pdcache_line_size == 0) {
1164 printf("MIPS32/64 params: no Dcache\n");
1165 } else {
1166 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1167 "ways=%d, sets=%d, colors=%d\n", "Dcache",
1168 mci->mci_pdcache_line_size,
1169 mci->mci_pdcache_way_size * mci->mci_pdcache_ways,
1170 mci->mci_pdcache_ways,
1171 mci->mci_pdcache_way_size / mci->mci_pdcache_line_size,
1172 mci->mci_pdcache_way_size >> PAGE_SHIFT);
1173 }
1174 #endif /* CACHE_DEBUG */
1175
1176 mco->mco_icache_sync_all = mipsNN_picache_sync_all;
1177 mco->mco_icache_sync_range = mipsNN_picache_sync_range;
1178 mco->mco_icache_sync_range_index = mipsNN_picache_sync_range_index;
1179
1180 switch (mci->mci_picache_line_size) {
1181 case 16:
1182 /* used internally by mipsNN_picache_sync_range */
1183 mco->mco_intern_icache_sync_range =
1184 cache_r4k_icache_hit_inv_16;
1185
1186 /* used internally by mipsNN_picache_sync_range_index */
1187 mco->mco_intern_icache_sync_range_index =
1188 cache_r4k_icache_index_inv_16;
1189 break;
1190 case 32:
1191 /* used internally by mipsNN_picache_sync_range */
1192 mco->mco_intern_icache_sync_range =
1193 cache_r4k_icache_hit_inv_32;
1194
1195 /* used internally by mipsNN_picache_sync_range_index */
1196 mco->mco_intern_icache_sync_range_index =
1197 cache_r4k_icache_index_inv_32;
1198 break;
1199 #ifdef MIPS_DISABLE_L1_CACHE
1200 case 0:
1201 mco->mco_icache_sync_all = no_cache_op;
1202 mco->mco_icache_sync_range = no_cache_op_range;
1203 mco->mco_icache_sync_range_index = no_cache_op_range_index;
1204 break;
1205 #endif
1206 case 64:
1207 /* used internally by mipsNN_picache_sync_range */
1208 mco->mco_intern_icache_sync_range =
1209 cache_r4k_icache_hit_inv_64;
1210
1211 /* used internally by mipsNN_picache_sync_range_index */
1212 mco->mco_intern_icache_sync_range_index =
1213 cache_r4k_icache_index_inv_64;
1214 break;
1215 case 128:
1216 #ifdef MIPS64_OCTEON
1217 mco->mco_icache_sync_all = octeon_icache_sync_all;
1218 mco->mco_icache_sync_range = octeon_icache_sync_range;
1219 mco->mco_icache_sync_range_index = octeon_icache_sync_range_index;
1220 #else
1221 /* used internally by mipsNN_picache_sync_range */
1222 mco->mco_intern_icache_sync_range =
1223 cache_r4k_icache_hit_inv_128;
1224
1225 /* used internally by mipsNN_picache_sync_range_index */
1226 mco->mco_intern_icache_sync_range_index =
1227 cache_r4k_icache_index_inv_128;
1228 #endif
1229 break;
1230 default:
1231 panic("no Icache ops for %dB lines",
1232 mci->mci_picache_line_size);
1233 }
1234
1235 mco->mco_pdcache_wbinv_all = mipsNN_pdcache_wbinv_all;
1236 mco->mco_pdcache_wbinv_range_index = mipsNN_pdcache_wbinv_range_index;
1237
1238 switch (mci->mci_pdcache_line_size) {
1239 case 16:
1240 mco->mco_pdcache_wbinv_range =
1241 cache_r4k_pdcache_hit_wb_inv_16;
1242 mco->mco_pdcache_inv_range =
1243 cache_r4k_pdcache_hit_inv_16;
1244 mco->mco_pdcache_wb_range =
1245 cache_r4k_pdcache_hit_wb_16;
1246
1247 /* used internally by mipsNN_pdcache_wbinv_range_index */
1248 mco->mco_intern_pdcache_wbinv_range_index =
1249 cache_r4k_pdcache_index_wb_inv_16;
1250 break;
1251 case 32:
1252 mco->mco_pdcache_wbinv_range =
1253 cache_r4k_pdcache_hit_wb_inv_32;
1254 mco->mco_pdcache_inv_range =
1255 cache_r4k_pdcache_hit_inv_32;
1256 mco->mco_pdcache_wb_range =
1257 cache_r4k_pdcache_hit_wb_32;
1258
1259 /* used internally by mipsNN_pdcache_wbinv_range_index */
1260 mco->mco_intern_pdcache_wbinv_range_index =
1261 cache_r4k_pdcache_index_wb_inv_32;
1262 break;
1263 case 64:
1264 mco->mco_pdcache_wbinv_range =
1265 cache_r4k_pdcache_hit_wb_inv_64;
1266 mco->mco_pdcache_inv_range =
1267 cache_r4k_pdcache_hit_inv_64;
1268 mco->mco_pdcache_wb_range =
1269 cache_r4k_pdcache_hit_wb_64;
1270
1271 /* used internally by mipsNN_pdcache_wbinv_range_index */
1272 mco->mco_intern_pdcache_wbinv_range_index =
1273 cache_r4k_pdcache_index_wb_inv_64;
1274 case 128:
1275 #ifdef MIPS64_OCTEON
1276 mco->mco_pdcache_wbinv_all = octeon_pdcache_inv_all;
1277 mco->mco_pdcache_wbinv_range = octeon_pdcache_inv_range;
1278 mco->mco_pdcache_wbinv_range_index = octeon_pdcache_inv_range_index;
1279 mco->mco_pdcache_inv_range = octeon_pdcache_inv_range;
1280 mco->mco_pdcache_wb_range = no_cache_op_range;
1281 #else
1282 mco->mco_pdcache_wbinv_range =
1283 cache_r4k_pdcache_hit_wb_inv_128;
1284 mco->mco_pdcache_inv_range =
1285 cache_r4k_pdcache_hit_inv_128;
1286 mco->mco_pdcache_wb_range =
1287 cache_r4k_pdcache_hit_wb_128;
1288
1289 /* used internally by mipsNN_pdcache_wbinv_range_index */
1290 mco->mco_intern_pdcache_wbinv_range_index =
1291 cache_r4k_pdcache_index_wb_inv_128;
1292 #endif
1293 break;
1294 #ifdef MIPS_DISABLE_L1_CACHE
1295 case 0:
1296 mco->mco_pdcache_wbinv_all = no_cache_op;
1297 mco->mco_pdcache_wbinv_range = no_cache_op_range;
1298 mco->mco_pdcache_wbinv_range_index = no_cache_op_index;
1299 mco->mco_pdcache_inv_range = no_cache_op_range;
1300 mco->mco_pdcache_wb_range = no_cache_op_range;
1301 break;
1302 #endif
1303 default:
1304 panic("no Dcache ops for %dB lines",
1305 mci->mci_pdcache_line_size);
1306 }
1307
1308 mco->mco_intern_pdcache_sync_all = mco->mco_pdcache_wbinv_all;
1309 mco->mco_intern_pdcache_sync_range_index = mco->mco_intern_pdcache_wbinv_range_index;
1310 mco->mco_intern_pdcache_sync_range = mco->mco_pdcache_wb_range;
1311
1312 if (MIPSNN_CFG1_M & cfg1) {
1313 uint32_t cfg2 = mipsNN_cp0_config2_read();
1314
1315 switch (MIPSNN_GET(CFG2_SL, cfg2)) {
1316 case MIPSNN_CFG2_SL_NONE:
1317 break;
1318 default:
1319 mci->mci_scache_unified = true;
1320
1321 mci->mci_sdcache_line_size = MIPSNN_CFG2_SL(cfg2);
1322 mci->mci_sdcache_way_size =
1323 mci->mci_sdcache_line_size * MIPSNN_CFG2_SS(cfg2);
1324 mci->mci_sdcache_ways = MIPSNN_CFG2_SA(cfg2) + 1;
1325
1326 /*
1327 * Compute the total size and "way mask" for the
1328 * secondary Dcache.
1329 */
1330 mci->mci_sdcache_size =
1331 mci->mci_sdcache_way_size * mci->mci_sdcache_ways;
1332 mci->mci_sdcache_way_mask =
1333 mci->mci_sdcache_way_size - 1;
1334
1335 /*
1336 * cache is unified so copy data info to inst info.
1337 */
1338 mci->mci_sicache_line_size = mci->mci_sdcache_line_size;
1339 mci->mci_sicache_way_size = mci->mci_sdcache_way_size;
1340 mci->mci_sicache_ways = mci->mci_sdcache_ways;
1341 mci->mci_sicache_size = mci->mci_sdcache_size;
1342 mci->mci_sicache_way_mask = mci->mci_sdcache_way_mask;
1343
1344 break;
1345 }
1346
1347 // Note we don't set up any sd cache ops because we expect that
1348 // the coherence checks below will overwrite them with no ops.
1349
1350 #ifdef CACHE_DEBUG
1351 if (mci->mci_sdcache_line_size != 0) {
1352 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1353 "ways=%d, sets=%d, colors=%d\n",
1354 "SDcache",
1355 mci->mci_sdcache_line_size,
1356 mci->mci_sdcache_way_size * mci->mci_sdcache_ways,
1357 mci->mci_sdcache_ways,
1358 mci->mci_sdcache_way_size
1359 / mci->mci_sdcache_line_size,
1360 mci->mci_sdcache_way_size >> PAGE_SHIFT);
1361 }
1362 #endif
1363
1364 switch (MIPSNN_GET(CFG2_TL, cfg2)) {
1365 case MIPSNN_CFG2_TL_NONE:
1366 break;
1367 default:
1368 mci->mci_tcache_line_size = MIPSNN_CFG2_TL(cfg2);
1369 mci->mci_tcache_way_size =
1370 mci->mci_tcache_line_size * MIPSNN_CFG2_TS(cfg2);
1371 mci->mci_tcache_ways = MIPSNN_CFG2_TA(cfg2) + 1;
1372
1373 /*
1374 * Compute the total size and "way mask" for the
1375 * secondary Dcache.
1376 */
1377 mci->mci_tcache_size =
1378 mci->mci_tcache_way_size * mci->mci_tcache_ways;
1379 mci->mci_tcache_way_mask =
1380 mci->mci_tcache_way_size - 1;
1381 break;
1382 }
1383 }
1384
1385 /*
1386 * calculate the alias masks and from them set to virtual alias flags.
1387 */
1388 mci->mci_cache_alias_mask = mci->mci_pdcache_way_mask & -PAGE_SIZE;
1389 mci->mci_cache_virtual_alias = (mci->mci_cache_alias_mask != 0);
1390
1391 mci->mci_icache_alias_mask = mci->mci_picache_way_mask & -PAGE_SIZE;
1392 mci->mci_icache_virtual_alias = (mci->mci_icache_alias_mask != 0);
1393
1394 /*
1395 * RMI (NetLogic/Broadcom) don't support WB (op 6) so we have to make
1396 * do with WBINV (op 5). This is merely for correctness since because
1397 * the caches are coherent, these routines will become noops in a bit.
1398 */
1399 if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_RMI) {
1400 mco->mco_pdcache_wb_range = mco->mco_pdcache_wbinv_range;
1401 mco->mco_intern_pdcache_sync_range = mco->mco_pdcache_wbinv_range;
1402 if (MIPSNN_GET(CFG_AR, cfg) == MIPSNN_CFG_AR_REV2) {
1403 mci->mci_pdcache_write_through = true;
1404 mci->mci_sdcache_write_through = false;
1405 KASSERT(PAGE_SIZE >= mci->mci_picache_way_size
1406 || MIPS_ICACHE_VIRTUAL_ALIAS);
1407 } else {
1408 KASSERT(MIPS_CACHE_VIRTUAL_ALIAS == 0);
1409 KASSERT(MIPS_ICACHE_VIRTUAL_ALIAS == 0);
1410 }
1411 } else if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_MTI) {
1412 /*
1413 * All MTI cores share a (mostly) common config7 defintion.
1414 * Use it to determine if the caches have virtual aliases.
1415 * If the core doesn't have a config7 register, its caches
1416 * are too small or have too many ways to have aliases.
1417 */
1418 if (opts->mips_cpu->cpu_cp0flags & MIPS_CP0FL_CONFIG7) {
1419 const uint32_t cfg7 = mipsNN_cp0_config7_read();
1420 if (cfg7 & MIPSNN_MTI_CFG7_AR) {
1421 /* [Data] Alias Removal Present */
1422 mci->mci_cache_virtual_alias = false;
1423 }
1424 if (cfg7 & MIPSNN_MTI_CFG7_IAR) {
1425 /* Instruction Alias Removal Present */
1426 mci->mci_icache_virtual_alias = false;
1427 }
1428 #if 0
1429 } else {
1430 KASSERT(mci->mci_pdcache_way_size <= PAGE_SIZE);
1431 KASSERT(mci->mci_picache_way_size <= PAGE_SIZE);
1432 #endif
1433 }
1434 }
1435
1436 mipsNN_cache_init(cfg, cfg1);
1437
1438 if (opts->mips_cpu_flags &
1439 (CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_I_D_CACHE_COHERENT)) {
1440 #ifdef CACHE_DEBUG
1441 printf(" Dcache is coherent\n");
1442 #endif
1443 mco->mco_pdcache_wbinv_all = no_cache_op;
1444 mco->mco_pdcache_wbinv_range = no_cache_op_range;
1445 mco->mco_pdcache_wbinv_range_index = no_cache_op_range_index;
1446 mco->mco_pdcache_inv_range = no_cache_op_range;
1447 mco->mco_pdcache_wb_range = no_cache_op_range;
1448 mco->mco_sdcache_wbinv_all = no_cache_op;
1449 mco->mco_sdcache_wbinv_range = no_cache_op_range;
1450 mco->mco_sdcache_wbinv_range_index = no_cache_op_range_index;
1451 mco->mco_sdcache_inv_range = no_cache_op_range;
1452 mco->mco_sdcache_wb_range = no_cache_op_range;
1453 }
1454 if (opts->mips_cpu_flags & CPU_MIPS_I_D_CACHE_COHERENT) {
1455 #ifdef CACHE_DEBUG
1456 printf(" Icache is coherent against Dcache\n");
1457 #endif
1458 mco->mco_intern_pdcache_sync_all = no_cache_op;
1459 mco->mco_intern_pdcache_sync_range_index = no_cache_op_range_index;
1460 mco->mco_intern_pdcache_sync_range = no_cache_op_range;
1461 }
1462 }
1463 #endif /* MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2 > 0 */
1464