1 /* $OpenBSD: cache_mips64r2.c,v 1.4 2022/08/29 02:08:13 jsg Exp $ */
2
3 /*
4 * Copyright (c) 2014 Miodrag Vallat.
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 /*
20 * Cache handling code for mips64r2 compatible processors
21 */
22
23 #include <sys/param.h>
24 #include <sys/systm.h>
25
26 #include <mips64/cache.h>
27 #include <mips64/mips_cpu.h>
28 #include <machine/cpu.h>
29
30 #include <uvm/uvm_extern.h>
31
32 #define IndexInvalidate_I 0x00
33 #define IndexWBInvalidate_D 0x01
34 #define IndexWBInvalidate_T 0x02
35 #define IndexWBInvalidate_S 0x03
36
37 #define HitInvalidate_D 0x11
38 #define HitInvalidate_T 0x12
39 #define HitInvalidate_S 0x13
40
41 #define HitWBInvalidate_D 0x15
42 #define HitWBInvalidate_T 0x16
43 #define HitWBInvalidate_S 0x17
44
45 #define cache(op,addr) \
46 __asm__ volatile \
47 ("cache %0, 0(%1)" :: "i"(op), "r"(addr) : "memory")
48
49 static __inline__ void mips64r2_hitinv_primary(vaddr_t, vsize_t, vsize_t);
50 static __inline__ void mips64r2_hitinv_secondary(vaddr_t, vsize_t, vsize_t);
51 static __inline__ void mips64r2_hitinv_ternary(vaddr_t, vsize_t, vsize_t);
52 static __inline__ void mips64r2_hitwbinv_primary(vaddr_t, vsize_t, vsize_t);
53 static __inline__ void mips64r2_hitwbinv_secondary(vaddr_t, vsize_t, vsize_t);
54 static __inline__ void mips64r2_hitwbinv_ternary(vaddr_t, vsize_t, vsize_t);
55
56 void
mips64r2_ConfigCache(struct cpu_info * ci)57 mips64r2_ConfigCache(struct cpu_info *ci)
58 {
59 uint32_t cfg, valias_mask;
60 uint32_t s, l, a;
61
62 cfg = cp0_get_config();
63 if ((cfg & 0x80000000) == 0)
64 panic("no M bit in cfg0.0");
65
66 cfg = cp0_get_config_1();
67
68 a = 1 + ((cfg & CONFIG1_DA) >> CONFIG1_DA_SHIFT);
69 l = (cfg & CONFIG1_DL) >> CONFIG1_DL_SHIFT;
70 s = (cfg & CONFIG1_DS) >> CONFIG1_DS_SHIFT;
71 ci->ci_l1data.linesize = 2 << l;
72 ci->ci_l1data.setsize = (64 << s) * ci->ci_l1data.linesize;
73 ci->ci_l1data.sets = a;
74 ci->ci_l1data.size = ci->ci_l1data.sets * ci->ci_l1data.setsize;
75
76 a = 1 + ((cfg & CONFIG1_IA) >> CONFIG1_IA_SHIFT);
77 l = (cfg & CONFIG1_IL) >> CONFIG1_IL_SHIFT;
78 s = (cfg & CONFIG1_IS) >> CONFIG1_IS_SHIFT;
79 ci->ci_l1inst.linesize = 2 << l;
80 ci->ci_l1inst.setsize = (64 << s) * ci->ci_l1inst.linesize;
81 ci->ci_l1inst.sets = a;
82 ci->ci_l1inst.size = ci->ci_l1inst.sets * ci->ci_l1inst.setsize;
83
84 memset(&ci->ci_l2, 0, sizeof(struct cache_info));
85 memset(&ci->ci_l3, 0, sizeof(struct cache_info));
86
87 if ((cfg & 0x80000000) != 0) {
88 cfg = cp0_get_config_2();
89
90 a = 1 + ((cfg >> 0) & 0x0f);
91 l = (cfg >> 4) & 0x0f;
92 s = (cfg >> 8) & 0x0f;
93 if (l != 0) {
94 ci->ci_l2.linesize = 2 << l;
95 ci->ci_l2.setsize = (64 << s) * ci->ci_l2.linesize;
96 ci->ci_l2.sets = a;
97 ci->ci_l2.size = ci->ci_l2.sets * ci->ci_l2.setsize;
98 }
99
100 a = 1 + ((cfg >> 16) & 0x0f);
101 l = (cfg >> 20) & 0x0f;
102 s = (cfg >> 24) & 0x0f;
103 if (l != 0) {
104 ci->ci_l3.linesize = 2 << l;
105 ci->ci_l3.setsize = (64 << s) * ci->ci_l3.linesize;
106 ci->ci_l3.sets = a;
107 ci->ci_l3.size = ci->ci_l3.sets * ci->ci_l3.setsize;
108 }
109 }
110
111 valias_mask = (max(ci->ci_l1inst.setsize, ci->ci_l1data.setsize) - 1) &
112 ~PAGE_MASK;
113
114 if (valias_mask != 0) {
115 valias_mask |= PAGE_MASK;
116 #ifdef MULTIPROCESSOR
117 if (valias_mask > cache_valias_mask) {
118 #endif
119 cache_valias_mask = valias_mask;
120 pmap_prefer_mask = valias_mask;
121 #ifdef MULTIPROCESSOR
122 }
123 #endif
124 }
125
126 ci->ci_SyncCache = mips64r2_SyncCache;
127 ci->ci_InvalidateICache = mips64r2_InvalidateICache;
128 ci->ci_InvalidateICachePage = mips64r2_InvalidateICachePage;
129 ci->ci_SyncICache = mips64r2_SyncICache;
130 ci->ci_SyncDCachePage = mips64r2_SyncDCachePage;
131 ci->ci_HitSyncDCachePage = mips64r2_HitSyncDCachePage;
132 ci->ci_HitSyncDCache = mips64r2_HitSyncDCache;
133 ci->ci_HitInvalidateDCache = mips64r2_HitInvalidateDCache;
134 ci->ci_IOSyncDCache = mips64r2_IOSyncDCache;
135 }
136
137 static __inline__ void
mips64r2_hitwbinv_primary(vaddr_t va,vsize_t sz,vsize_t line)138 mips64r2_hitwbinv_primary(vaddr_t va, vsize_t sz, vsize_t line)
139 {
140 vaddr_t eva;
141
142 eva = va + sz;
143 while (va != eva) {
144 cache(HitWBInvalidate_D, va);
145 va += line;
146 }
147 }
148
149 static __inline__ void
mips64r2_hitwbinv_secondary(vaddr_t va,vsize_t sz,vsize_t line)150 mips64r2_hitwbinv_secondary(vaddr_t va, vsize_t sz, vsize_t line)
151 {
152 vaddr_t eva;
153
154 eva = va + sz;
155 while (va != eva) {
156 cache(HitWBInvalidate_S, va);
157 va += line;
158 }
159 }
160
161 static __inline__ void
mips64r2_hitwbinv_ternary(vaddr_t va,vsize_t sz,vsize_t line)162 mips64r2_hitwbinv_ternary(vaddr_t va, vsize_t sz, vsize_t line)
163 {
164 vaddr_t eva;
165
166 eva = va + sz;
167 while (va != eva) {
168 cache(HitWBInvalidate_T, va);
169 va += line;
170 }
171 }
172
173 static __inline__ void
mips64r2_hitinv_primary(vaddr_t va,vsize_t sz,vsize_t line)174 mips64r2_hitinv_primary(vaddr_t va, vsize_t sz, vsize_t line)
175 {
176 vaddr_t eva;
177
178 eva = va + sz;
179 while (va != eva) {
180 cache(HitInvalidate_D, va);
181 va += line;
182 }
183 }
184
185 static __inline__ void
mips64r2_hitinv_secondary(vaddr_t va,vsize_t sz,vsize_t line)186 mips64r2_hitinv_secondary(vaddr_t va, vsize_t sz, vsize_t line)
187 {
188 vaddr_t eva;
189
190 eva = va + sz;
191 while (va != eva) {
192 cache(HitInvalidate_S, va);
193 va += line;
194 }
195 }
196
197 static __inline__ void
mips64r2_hitinv_ternary(vaddr_t va,vsize_t sz,vsize_t line)198 mips64r2_hitinv_ternary(vaddr_t va, vsize_t sz, vsize_t line)
199 {
200 vaddr_t eva;
201
202 eva = va + sz;
203 while (va != eva) {
204 cache(HitInvalidate_T, va);
205 va += line;
206 }
207 }
208
209 /*
210 * Writeback and invalidate all caches.
211 */
212 void
mips64r2_SyncCache(struct cpu_info * ci)213 mips64r2_SyncCache(struct cpu_info *ci)
214 {
215 vaddr_t sva, eva;
216
217 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
218 eva = sva + ci->ci_l1inst.linesize;
219 while (sva != eva) {
220 cache(IndexInvalidate_I, sva);
221 sva += ci->ci_l1inst.linesize;
222 }
223
224 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
225 eva = sva + ci->ci_l1data.linesize;
226 while (sva != eva) {
227 cache(IndexWBInvalidate_D, sva);
228 sva += ci->ci_l1data.linesize;
229 }
230
231 if (ci->ci_l2.size != 0) {
232 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
233 eva = sva + ci->ci_l2.size;
234 while (sva != eva) {
235 cache(IndexWBInvalidate_S, sva);
236 sva += ci->ci_l2.linesize;
237 }
238 }
239
240 if (ci->ci_l3.size != 0) {
241 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
242 eva = sva + ci->ci_l3.size;
243 while (sva != eva) {
244 cache(IndexWBInvalidate_T, sva);
245 sva += ci->ci_l3.linesize;
246 }
247 }
248 }
249
250 /*
251 * Invalidate I$ for the given range.
252 */
253 void
mips64r2_InvalidateICache(struct cpu_info * ci,vaddr_t _va,size_t _sz)254 mips64r2_InvalidateICache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
255 {
256 vaddr_t va, sva, eva, iva;
257 vsize_t sz, offs;
258 uint set, nsets;
259
260 /* extend the range to integral cache lines */
261 va = _va & ~(ci->ci_l1inst.linesize - 1);
262 sz = ((_va + _sz + ci->ci_l1inst.linesize - 1) & ~(ci->ci_l1inst.linesize - 1)) - va;
263
264 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
265 offs = ci->ci_l1inst.setsize;
266 nsets = ci->ci_l1inst.sets;
267 /* keep only the index bits */
268 sva |= va & (offs - 1);
269 eva = sva + sz;
270
271 while (sva != eva) {
272 for (set = nsets, iva = sva; set != 0; set--, iva += offs)
273 cache(IndexInvalidate_I, iva);
274 sva += ci->ci_l1inst.linesize;
275 }
276 }
277
278 /*
279 * Register a given page for I$ invalidation.
280 */
281 void
mips64r2_InvalidateICachePage(struct cpu_info * ci,vaddr_t va)282 mips64r2_InvalidateICachePage(struct cpu_info *ci, vaddr_t va)
283 {
284 /* this code is too generic to allow for lazy I$ invalidates, yet */
285 mips64r2_InvalidateICache(ci, va, PAGE_SIZE);
286 }
287
288 /*
289 * Perform postponed I$ invalidation.
290 */
291 void
mips64r2_SyncICache(struct cpu_info * ci)292 mips64r2_SyncICache(struct cpu_info *ci)
293 {
294 }
295
296 /*
297 * Writeback D$ for the given page.
298 */
299 void
mips64r2_SyncDCachePage(struct cpu_info * ci,vaddr_t va,paddr_t pa)300 mips64r2_SyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa)
301 {
302 vaddr_t sva, eva, iva;
303 vsize_t line, offs;
304 uint set, nsets;
305
306 line = ci->ci_l1data.linesize;
307 sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
308 offs = ci->ci_l1data.setsize;
309 nsets = ci->ci_l1data.sets;
310 /* keep only the index bits */
311 sva += va & (offs - 1);
312 eva = sva + PAGE_SIZE;
313 while (sva != eva) {
314 for (set = nsets, iva = sva; set != 0; set--, iva += offs)
315 cache(IndexWBInvalidate_D, iva);
316 sva += ci->ci_l1data.linesize;
317 }
318 }
319
320 /*
321 * Writeback D$ for the given page, which is expected to be currently
322 * mapped, allowing the use of `Hit' operations. This is less aggressive
323 * than using `Index' operations.
324 */
325
326 void
mips64r2_HitSyncDCachePage(struct cpu_info * ci,vaddr_t va,paddr_t pa)327 mips64r2_HitSyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa)
328 {
329 mips64r2_hitwbinv_primary(va, PAGE_SIZE, ci->ci_l1data.linesize);
330 }
331
332 /*
333 * Writeback D$ for the given range. Range is expected to be currently
334 * mapped, allowing the use of `Hit' operations. This is less aggressive
335 * than using `Index' operations.
336 */
337
338 void
mips64r2_HitSyncDCache(struct cpu_info * ci,vaddr_t _va,size_t _sz)339 mips64r2_HitSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
340 {
341 vaddr_t va;
342 vsize_t sz;
343
344 /* extend the range to integral cache lines */
345 va = _va & ~(ci->ci_l1data.linesize - 1);
346 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
347 mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize);
348 }
349
350 /*
351 * Invalidate D$ for the given range. Range is expected to be currently
352 * mapped, allowing the use of `Hit' operations. This is less aggressive
353 * than using `Index' operations.
354 */
355
356 void
mips64r2_HitInvalidateDCache(struct cpu_info * ci,vaddr_t _va,size_t _sz)357 mips64r2_HitInvalidateDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
358 {
359 vaddr_t va;
360 vsize_t sz;
361
362 /* extend the range to integral cache lines */
363 va = _va & ~(ci->ci_l1data.linesize - 1);
364 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
365 mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize);
366 }
367
368 /*
369 * Backend for bus_dmamap_sync(). Enforce coherency of the given range
370 * by performing the necessary cache writeback and/or invalidate
371 * operations.
372 */
373 void
mips64r2_IOSyncDCache(struct cpu_info * ci,vaddr_t _va,size_t _sz,int how)374 mips64r2_IOSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz, int how)
375 {
376 vaddr_t va;
377 vsize_t sz;
378 int partial_start, partial_end;
379
380 /*
381 * L1
382 */
383
384 /* extend the range to integral cache lines */
385 va = _va & ~(ci->ci_l1data.linesize - 1);
386 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
387
388 switch (how) {
389 case CACHE_SYNC_R:
390 /* writeback partial cachelines */
391 if (((_va | _sz) & (ci->ci_l1data.linesize - 1)) != 0) {
392 partial_start = va != _va;
393 partial_end = va + sz != _va + _sz;
394 } else {
395 partial_start = partial_end = 0;
396 }
397 if (partial_start) {
398 cache(HitWBInvalidate_D, va);
399 va += ci->ci_l1data.linesize;
400 sz -= ci->ci_l1data.linesize;
401 }
402 if (sz != 0 && partial_end) {
403 sz -= ci->ci_l1data.linesize;
404 cache(HitWBInvalidate_D, va + sz);
405 }
406 if (sz != 0)
407 mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize);
408 break;
409 case CACHE_SYNC_X:
410 case CACHE_SYNC_W:
411 mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize);
412 break;
413 }
414
415 /*
416 * L2
417 */
418
419 if (ci->ci_l2.size != 0) {
420 /* extend the range to integral cache lines */
421 va = _va & ~(ci->ci_l2.linesize - 1);
422 sz = ((_va + _sz + ci->ci_l2.linesize - 1) & ~(ci->ci_l2.linesize - 1)) - va;
423
424 switch (how) {
425 case CACHE_SYNC_R:
426 /* writeback partial cachelines */
427 if (((_va | _sz) & (ci->ci_l2.linesize - 1)) != 0) {
428 partial_start = va != _va;
429 partial_end = va + sz != _va + _sz;
430 } else {
431 partial_start = partial_end = 0;
432 }
433 if (partial_start) {
434 cache(HitWBInvalidate_S, va);
435 va += ci->ci_l2.linesize;
436 sz -= ci->ci_l2.linesize;
437 }
438 if (sz != 0 && partial_end) {
439 sz -= ci->ci_l2.linesize;
440 cache(HitWBInvalidate_S, va + sz);
441 }
442 if (sz != 0)
443 mips64r2_hitinv_secondary(va, sz, ci->ci_l2.linesize);
444 break;
445 case CACHE_SYNC_X:
446 case CACHE_SYNC_W:
447 mips64r2_hitwbinv_secondary(va, sz, ci->ci_l2.linesize);
448 break;
449 }
450 }
451
452 /*
453 * L3
454 */
455
456 if (ci->ci_l3.size != 0) {
457 /* extend the range to integral cache lines */
458 va = _va & ~(ci->ci_l3.linesize - 1);
459 sz = ((_va + _sz + ci->ci_l3.linesize - 1) & ~(ci->ci_l3.linesize - 1)) - va;
460
461 switch (how) {
462 case CACHE_SYNC_R:
463 /* writeback partial cachelines */
464 if (((_va | _sz) & (ci->ci_l3.linesize - 1)) != 0) {
465 partial_start = va != _va;
466 partial_end = va + sz != _va + _sz;
467 } else {
468 partial_start = partial_end = 0;
469 }
470 if (partial_start) {
471 cache(HitWBInvalidate_S, va);
472 va += ci->ci_l3.linesize;
473 sz -= ci->ci_l3.linesize;
474 }
475 if (sz != 0 && partial_end) {
476 sz -= ci->ci_l3.linesize;
477 cache(HitWBInvalidate_S, va + sz);
478 }
479 if (sz != 0)
480 mips64r2_hitinv_ternary(va, sz, ci->ci_l3.linesize);
481 break;
482 case CACHE_SYNC_X:
483 case CACHE_SYNC_W:
484 mips64r2_hitwbinv_ternary(va, sz, ci->ci_l3.linesize);
485 break;
486 }
487 }
488 }
489