1 /* $NetBSD: booke_machdep.c,v 1.33 2021/03/30 14:29:54 rin Exp $ */
2 /*-
3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
8 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
9 *
10 * This material is based upon work supported by the Defense Advanced Research
11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
12 * Contract No. N66001-09-C-2073.
13 * Approved for Public Release, Distribution Unlimited
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #define __INTR_PRIVATE
38 #define _POWERPC_BUS_DMA_PRIVATE
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: booke_machdep.c,v 1.33 2021/03/30 14:29:54 rin Exp $");
42
43 #include "ksyms.h"
44
45 #ifdef _KERNEL_OPT
46 #include "opt_ddb.h"
47 #include "opt_modular.h"
48 #include "opt_multiprocessor.h"
49 #endif
50
51 #include <sys/param.h>
52 #include <sys/cpu.h>
53 #include <sys/device.h>
54 #include <sys/intr.h>
55 #include <sys/mount.h>
56 #include <sys/msgbuf.h>
57 #include <sys/kernel.h>
58 #include <sys/reboot.h>
59 #include <sys/bus.h>
60 #include <sys/cpu.h>
61
62 #include <uvm/uvm_extern.h>
63
64 #include <dev/cons.h>
65
66 #include <powerpc/pcb.h>
67 #include <powerpc/spr.h>
68 #include <powerpc/booke/spr.h>
69 #include <powerpc/booke/cpuvar.h>
70
71 /*
72 * Global variables used here and there
73 */
74 paddr_t msgbuf_paddr;
75 psize_t pmemsize;
76 struct vm_map *phys_map;
77
78 #ifdef MODULAR
79 register_t cpu_psluserset = PSL_USERSET;
80 register_t cpu_pslusermod = PSL_USERMOD;
81 register_t cpu_pslusermask = PSL_USERMASK;
82 #endif
83
84 static bus_addr_t booke_dma_phys_to_bus_mem(bus_dma_tag_t, bus_addr_t);
85 static bus_addr_t booke_dma_bus_mem_to_phys(bus_dma_tag_t, bus_addr_t);
86
87
88 struct powerpc_bus_dma_tag booke_bus_dma_tag = {
89 ._dmamap_create = _bus_dmamap_create,
90 ._dmamap_destroy = _bus_dmamap_destroy,
91 ._dmamap_load = _bus_dmamap_load,
92 ._dmamap_load_mbuf = _bus_dmamap_load_mbuf,
93 ._dmamap_load_uio = _bus_dmamap_load_uio,
94 ._dmamap_load_raw = _bus_dmamap_load_raw,
95 ._dmamap_unload = _bus_dmamap_unload,
96 /*
97 * The caches on BookE are coherent so we don't need to do any special
98 * cache synchronization.
99 */
100 //._dmamap_sync = _bus_dmamap_sync,
101 ._dmamem_alloc = _bus_dmamem_alloc,
102 ._dmamem_free = _bus_dmamem_free,
103 ._dmamem_map = _bus_dmamem_map,
104 ._dmamem_unmap = _bus_dmamem_unmap,
105 ._dmamem_mmap = _bus_dmamem_mmap,
106 ._dma_phys_to_bus_mem = booke_dma_phys_to_bus_mem,
107 ._dma_bus_mem_to_phys = booke_dma_bus_mem_to_phys,
108 };
109
110 static bus_addr_t
booke_dma_phys_to_bus_mem(bus_dma_tag_t t,bus_addr_t a)111 booke_dma_phys_to_bus_mem(bus_dma_tag_t t, bus_addr_t a)
112 {
113 return a;
114 }
115
116 static bus_addr_t
booke_dma_bus_mem_to_phys(bus_dma_tag_t t,bus_addr_t a)117 booke_dma_bus_mem_to_phys(bus_dma_tag_t t, bus_addr_t a)
118 {
119 return a;
120 }
121
122 struct cpu_md_ops cpu_md_ops;
123
124 struct cpu_softc cpu_softc[] = {
125 [0] = {
126 .cpu_ci = &cpu_info[0],
127 },
128 #ifdef MULTIPROCESSOR
129 [CPU_MAXNUM-1] = {
130 .cpu_ci = &cpu_info[CPU_MAXNUM-1],
131 },
132 #endif
133 };
134 struct cpu_info cpu_info[] = {
135 [0] = {
136 .ci_curlwp = &lwp0,
137 .ci_tlb_info = &pmap_tlb0_info,
138 .ci_softc = &cpu_softc[0],
139 .ci_cpl = IPL_HIGH,
140 .ci_idepth = -1,
141 .ci_pmap_kern_segtab = &pmap_kern_segtab,
142 },
143 #ifdef MULTIPROCESSOR
144 [CPU_MAXNUM-1] = {
145 .ci_curlwp = NULL,
146 .ci_tlb_info = &pmap_tlb0_info,
147 .ci_softc = &cpu_softc[CPU_MAXNUM-1],
148 .ci_cpl = IPL_HIGH,
149 .ci_idepth = -1,
150 .ci_pmap_kern_segtab = &pmap_kern_segtab,
151 },
152 #endif
153 };
154 __CTASSERT(__arraycount(cpu_info) == __arraycount(cpu_softc));
155
156 /*
157 * This should probably be in autoconf! XXX
158 */
159 char machine[] = MACHINE; /* from <machine/param.h> */
160 char machine_arch[] = MACHINE_ARCH; /* from <machine/param.h> */
161
162 char bootpath[256];
163
164 #if NKSYMS || defined(DDB) || defined(MODULAR)
165 void *startsym, *endsym;
166 #endif
167
168 #if defined(MULTIPROCESSOR)
169 volatile struct cpu_hatch_data cpu_hatch_data __cacheline_aligned;
170 #endif
171
172 int fake_mapiodev = 1;
173
174 void
booke_cpu_startup(const char * model)175 booke_cpu_startup(const char *model)
176 {
177 vaddr_t minaddr, maxaddr;
178 char pbuf[9];
179
180 cpu_setmodel("%s", model);
181
182 printf("%s%s", copyright, version);
183
184 format_bytes(pbuf, sizeof(pbuf), ctob((uint64_t)physmem));
185 printf("total memory = %s\n", pbuf);
186
187 minaddr = 0;
188 /*
189 * Allocate a submap for physio
190 */
191 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
192 VM_PHYS_SIZE, 0, false, NULL);
193
194 /*
195 * No need to allocate an mbuf cluster submap. Mbuf clusters
196 * are allocated via the pool allocator, and we use direct-mapped
197 * pool pages.
198 */
199
200 format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false)));
201 printf("avail memory = %s\n", pbuf);
202
203 /*
204 * Register the tlb's evcnts
205 */
206 pmap_tlb_info_evcnt_attach(curcpu()->ci_tlb_info);
207
208 /*
209 * Set up the board properties database.
210 */
211 board_info_init();
212
213 /*
214 * Now that we have VM, malloc()s are OK in bus_space.
215 */
216 bus_space_mallocok();
217 fake_mapiodev = 0;
218
219 #ifdef MULTIPROCESSOR
220 pmap_kernel()->pm_active = kcpuset_running;
221 pmap_kernel()->pm_onproc = kcpuset_running;
222
223 for (size_t i = 1; i < __arraycount(cpu_info); i++) {
224 struct cpu_info * const ci = &cpu_info[i];
225 struct cpu_softc * const cpu = &cpu_softc[i];
226 cpu->cpu_ci = ci;
227 cpu->cpu_bst = cpu_softc[0].cpu_bst;
228 cpu->cpu_le_bst = cpu_softc[0].cpu_le_bst;
229 cpu->cpu_bsh = cpu_softc[0].cpu_bsh;
230 cpu->cpu_highmem = cpu_softc[0].cpu_highmem;
231 ci->ci_softc = cpu;
232 ci->ci_tlb_info = &pmap_tlb0_info;
233 ci->ci_cpl = IPL_HIGH;
234 ci->ci_idepth = -1;
235 ci->ci_pmap_kern_segtab = curcpu()->ci_pmap_kern_segtab;
236 }
237
238 kcpuset_create(&cpuset_info.cpus_running, true);
239 kcpuset_create(&cpuset_info.cpus_hatched, true);
240 kcpuset_create(&cpuset_info.cpus_paused, true);
241 kcpuset_create(&cpuset_info.cpus_resumed, true);
242 kcpuset_create(&cpuset_info.cpus_halted, true);
243
244 kcpuset_set(cpuset_info.cpus_running, cpu_number());
245 #endif /* MULTIPROCESSOR */
246 }
247
248 static void
dumpsys(void)249 dumpsys(void)
250 {
251
252 printf("dumpsys: TBD\n");
253 }
254
255 /*
256 * Halt or reboot the machine after syncing/dumping according to howto.
257 */
258 void
cpu_reboot(int howto,char * what)259 cpu_reboot(int howto, char *what)
260 {
261 static int syncing;
262 static char str[256];
263 char *ap = str, *ap1 = ap;
264
265 boothowto = howto;
266 if (!cold && !(howto & RB_NOSYNC) && !syncing) {
267 syncing = 1;
268 vfs_shutdown(); /* sync */
269 resettodr(); /* set wall clock */
270 }
271
272 splhigh();
273
274 if (!cold && (howto & RB_DUMP))
275 dumpsys();
276
277 doshutdownhooks();
278
279 pmf_system_shutdown(boothowto);
280
281 if ((howto & RB_POWERDOWN) == RB_POWERDOWN) {
282 /* Power off here if we know how...*/
283 }
284
285 if (howto & RB_HALT) {
286 printf("The operating system has halted.\n"
287 "Press any key to reboot.\n\n");
288 cnpollc(1); /* For proper keyboard command handling */
289 cngetc();
290 cnpollc(0);
291 }
292
293 printf("rebooting\n\n");
294 if (what && *what) {
295 if (strlen(what) > sizeof str - 5)
296 printf("boot string too large, ignored\n");
297 else {
298 strcpy(str, what);
299 ap1 = ap = str + strlen(str);
300 *ap++ = ' ';
301 }
302 }
303 *ap++ = '-';
304 if (howto & RB_SINGLE)
305 *ap++ = 's';
306 if (howto & RB_KDB)
307 *ap++ = 'd';
308 *ap++ = 0;
309 if (ap[-2] == '-')
310 *ap1 = 0;
311
312 /* flush cache for msgbuf */
313 dcache_wb(msgbuf_paddr, round_page(MSGBUFSIZE));
314
315 __asm volatile("msync; isync");
316 (*cpu_md_ops.md_cpu_reset)();
317
318 printf("%s: md_cpu_reset() failed!\n", __func__);
319 #ifdef DDB
320 for (;;)
321 Debugger();
322 #else
323 for (;;)
324 /* nothing */;
325 #endif
326 }
327
328 /*
329 * mapiodev:
330 *
331 * Allocate vm space and mapin the I/O address. Use reserved TLB
332 * mapping if one is found.
333 */
334 void *
mapiodev(paddr_t pa,psize_t len,bool prefetchable)335 mapiodev(paddr_t pa, psize_t len, bool prefetchable)
336 {
337 const vsize_t off = pa & PAGE_MASK;
338
339 /*
340 * See if we have reserved TLB entry for the pa. This needs to be
341 * true for console as we can't use uvm during early bootstrap.
342 */
343 void * const p = tlb_mapiodev(pa, len, prefetchable);
344 if (p != NULL)
345 return p;
346
347 if (fake_mapiodev)
348 panic("mapiodev: no TLB entry reserved for %llx+%llx",
349 (long long)pa, (long long)len);
350
351 const paddr_t orig_pa = pa;
352 const psize_t orig_len = len;
353 vsize_t align = 0;
354 pa = trunc_page(pa);
355 len = round_page(off + len);
356 /*
357 * If we are allocating a large amount (>= 1MB) try to get an
358 * aligned VA region for it so try to do a large mapping for it.
359 */
360 if ((len & (len - 1)) == 0 && len >= 0x100000)
361 align = len;
362
363 vaddr_t va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY);
364
365 if (va == 0 && align > 0) {
366 /*
367 * Large aligned request failed. Let's just get anything.
368 */
369 align = 0;
370 va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY);
371 }
372 if (va == 0)
373 return NULL;
374
375 if (align) {
376 /*
377 * Now try to map that via one big TLB entry.
378 */
379 pt_entry_t pte = pte_make_kenter_pa(pa, NULL,
380 VM_PROT_READ|VM_PROT_WRITE,
381 prefetchable ? 0 : PMAP_NOCACHE);
382 if (!tlb_ioreserve(va, len, pte)) {
383 void * const p0 = tlb_mapiodev(orig_pa, orig_len,
384 prefetchable);
385 KASSERT(p0 != NULL);
386 return p0;
387 }
388 }
389
390 for (va += len, pa += len; len > 0; len -= PAGE_SIZE) {
391 va -= PAGE_SIZE;
392 pa -= PAGE_SIZE;
393 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE,
394 prefetchable ? 0 : PMAP_NOCACHE);
395 }
396 pmap_update(pmap_kernel());
397 return (void *)(va + off);
398 }
399
400 void
unmapiodev(vaddr_t va,vsize_t len)401 unmapiodev(vaddr_t va, vsize_t len)
402 {
403 /* Nothing to do for reserved (ie. not uvm_km_alloc'd) mappings. */
404 if (va < VM_MIN_KERNEL_ADDRESS || va > VM_MAX_KERNEL_ADDRESS) {
405 tlb_unmapiodev(va, len);
406 return;
407 }
408
409 len = round_page((va & PAGE_MASK) + len);
410 va = trunc_page(va);
411
412 pmap_kremove(va, len);
413 uvm_km_free(kernel_map, va, len, UVM_KMF_VAONLY);
414 }
415
416 void
cpu_evcnt_attach(struct cpu_info * ci)417 cpu_evcnt_attach(struct cpu_info *ci)
418 {
419 struct cpu_softc * const cpu = ci->ci_softc;
420 const char * const xname = ci->ci_data.cpu_name;
421
422 evcnt_attach_dynamic_nozero(&ci->ci_ev_clock, EVCNT_TYPE_INTR,
423 NULL, xname, "clock");
424 evcnt_attach_dynamic_nozero(&cpu->cpu_ev_late_clock, EVCNT_TYPE_INTR,
425 NULL, xname, "late clock");
426 evcnt_attach_dynamic_nozero(&cpu->cpu_ev_exec_trap_sync, EVCNT_TYPE_TRAP,
427 NULL, xname, "exec pages synced (trap)");
428 evcnt_attach_dynamic_nozero(&ci->ci_ev_traps, EVCNT_TYPE_TRAP,
429 NULL, xname, "traps");
430 evcnt_attach_dynamic_nozero(&ci->ci_ev_kdsi, EVCNT_TYPE_TRAP,
431 &ci->ci_ev_traps, xname, "kernel DSI traps");
432 evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi, EVCNT_TYPE_TRAP,
433 &ci->ci_ev_traps, xname, "user DSI traps");
434 evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi_fatal, EVCNT_TYPE_TRAP,
435 &ci->ci_ev_udsi, xname, "user DSI failures");
436 evcnt_attach_dynamic_nozero(&ci->ci_ev_kisi, EVCNT_TYPE_TRAP,
437 &ci->ci_ev_traps, xname, "kernel ISI traps");
438 evcnt_attach_dynamic_nozero(&ci->ci_ev_isi, EVCNT_TYPE_TRAP,
439 &ci->ci_ev_traps, xname, "user ISI traps");
440 evcnt_attach_dynamic_nozero(&ci->ci_ev_isi_fatal, EVCNT_TYPE_TRAP,
441 &ci->ci_ev_isi, xname, "user ISI failures");
442 evcnt_attach_dynamic_nozero(&ci->ci_ev_scalls, EVCNT_TYPE_TRAP,
443 &ci->ci_ev_traps, xname, "system call traps");
444 evcnt_attach_dynamic_nozero(&ci->ci_ev_pgm, EVCNT_TYPE_TRAP,
445 &ci->ci_ev_traps, xname, "PGM traps");
446 evcnt_attach_dynamic_nozero(&ci->ci_ev_debug, EVCNT_TYPE_TRAP,
447 &ci->ci_ev_traps, xname, "debug traps");
448 evcnt_attach_dynamic_nozero(&ci->ci_ev_fpu, EVCNT_TYPE_TRAP,
449 &ci->ci_ev_traps, xname, "FPU unavailable traps");
450 evcnt_attach_dynamic_nozero(&ci->ci_ev_fpusw, EVCNT_TYPE_MISC,
451 &ci->ci_ev_fpu, xname, "FPU context switches");
452 evcnt_attach_dynamic_nozero(&ci->ci_ev_ali, EVCNT_TYPE_TRAP,
453 &ci->ci_ev_traps, xname, "user alignment traps");
454 evcnt_attach_dynamic_nozero(&ci->ci_ev_ali_fatal, EVCNT_TYPE_TRAP,
455 &ci->ci_ev_ali, xname, "user alignment traps");
456 evcnt_attach_dynamic_nozero(&ci->ci_ev_umchk, EVCNT_TYPE_TRAP,
457 &ci->ci_ev_umchk, xname, "user MCHK failures");
458 evcnt_attach_dynamic_nozero(&ci->ci_ev_vec, EVCNT_TYPE_TRAP,
459 &ci->ci_ev_traps, xname, "SPE unavailable");
460 evcnt_attach_dynamic_nozero(&ci->ci_ev_vecsw, EVCNT_TYPE_MISC,
461 &ci->ci_ev_vec, xname, "SPE context switches");
462 evcnt_attach_dynamic_nozero(&ci->ci_ev_ipi, EVCNT_TYPE_INTR,
463 NULL, xname, "IPIs");
464 evcnt_attach_dynamic_nozero(&ci->ci_ev_tlbmiss_soft, EVCNT_TYPE_TRAP,
465 &ci->ci_ev_traps, xname, "soft tlb misses");
466 evcnt_attach_dynamic_nozero(&ci->ci_ev_dtlbmiss_hard, EVCNT_TYPE_TRAP,
467 &ci->ci_ev_traps, xname, "data tlb misses");
468 evcnt_attach_dynamic_nozero(&ci->ci_ev_itlbmiss_hard, EVCNT_TYPE_TRAP,
469 &ci->ci_ev_traps, xname, "inst tlb misses");
470 }
471
472 #ifdef MULTIPROCESSOR
473 register_t
cpu_hatch(void)474 cpu_hatch(void)
475 {
476 struct cpuset_info * const csi = &cpuset_info;
477 const size_t id = cpu_number();
478
479 /*
480 * We've hatched so tell the spinup code.
481 */
482 kcpuset_set(csi->cpus_hatched, id);
483
484 /*
485 * Loop until running bit for this cpu is set.
486 */
487 while (!kcpuset_isset(csi->cpus_running, id)) {
488 continue;
489 }
490
491 /*
492 * Now that we are active, start the clocks.
493 */
494 cpu_initclocks();
495
496 /*
497 * Return sp of the idlelwp. Which we should be already using but ...
498 */
499 return curcpu()->ci_curpcb->pcb_sp;
500 }
501
502 void
cpu_boot_secondary_processors(void)503 cpu_boot_secondary_processors(void)
504 {
505 volatile struct cpuset_info * const csi = &cpuset_info;
506 CPU_INFO_ITERATOR cii;
507 struct cpu_info *ci;
508 kcpuset_t *running;
509
510 kcpuset_create(&running, true);
511
512 for (CPU_INFO_FOREACH(cii, ci)) {
513 /*
514 * Skip this CPU if it didn't successfully hatch.
515 */
516 if (!kcpuset_isset(csi->cpus_hatched, cpu_index(ci)))
517 continue;
518
519 KASSERT(!CPU_IS_PRIMARY(ci));
520 KASSERT(ci->ci_data.cpu_idlelwp);
521
522 kcpuset_set(running, cpu_index(ci));
523 }
524 KASSERT(kcpuset_match(csi->cpus_hatched, running));
525 if (!kcpuset_iszero(running)) {
526 kcpuset_merge(csi->cpus_running, running);
527 }
528 kcpuset_destroy(running);
529 }
530 #endif
531
532 uint32_t
cpu_read_4(bus_addr_t a)533 cpu_read_4(bus_addr_t a)
534 {
535 struct cpu_softc * const cpu = curcpu()->ci_softc;
536 // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a);
537 return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh, a);
538 }
539
540 uint8_t
cpu_read_1(bus_addr_t a)541 cpu_read_1(bus_addr_t a)
542 {
543 struct cpu_softc * const cpu = curcpu()->ci_softc;
544 // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a);
545 return bus_space_read_1(cpu->cpu_bst, cpu->cpu_bsh, a);
546 }
547
548 void
cpu_write_4(bus_addr_t a,uint32_t v)549 cpu_write_4(bus_addr_t a, uint32_t v)
550 {
551 struct cpu_softc * const cpu = curcpu()->ci_softc;
552 bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh, a, v);
553 }
554
555 void
cpu_write_1(bus_addr_t a,uint8_t v)556 cpu_write_1(bus_addr_t a, uint8_t v)
557 {
558 struct cpu_softc * const cpu = curcpu()->ci_softc;
559 bus_space_write_1(cpu->cpu_bst, cpu->cpu_bsh, a, v);
560 }
561
562 void
booke_sstep(struct trapframe * tf)563 booke_sstep(struct trapframe *tf)
564 {
565 uint32_t insn;
566
567 KASSERT(tf->tf_srr1 & PSL_DE);
568 if (ufetch_32((const void *)tf->tf_srr0, &insn) != 0)
569 return;
570
571 register_t dbcr0 = DBCR0_IAC1 | DBCR0_IDM;
572 register_t dbcr1 = DBCR1_IAC1US_USER | DBCR1_IAC1ER_DS1;
573 if ((insn >> 28) == 4) {
574 uint32_t iac2 = 0;
575 if ((insn >> 26) == 0x12) {
576 const int32_t off = (((int32_t)insn << 6) >> 6) & ~3;
577 iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off;
578 dbcr0 |= DBCR0_IAC2;
579 } else if ((insn >> 26) == 0x10) {
580 const int16_t off = insn & ~3;
581 iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off;
582 dbcr0 |= DBCR0_IAC2;
583 } else if ((insn & 0xfc00fffe) == 0x4c000420) {
584 iac2 = tf->tf_ctr;
585 dbcr0 |= DBCR0_IAC2;
586 } else if ((insn & 0xfc00fffe) == 0x4c000020) {
587 iac2 = tf->tf_lr;
588 dbcr0 |= DBCR0_IAC2;
589 }
590 if (dbcr0 & DBCR0_IAC2) {
591 dbcr1 |= DBCR1_IAC2US_USER | DBCR1_IAC2ER_DS1;
592 mtspr(SPR_IAC2, iac2);
593 }
594 }
595 mtspr(SPR_IAC1, tf->tf_srr0 + 4);
596 mtspr(SPR_DBCR1, dbcr1);
597 mtspr(SPR_DBCR0, dbcr0);
598 }
599
600 #ifdef DIAGNOSTIC
601 static inline void
swap_data(uint64_t * data,size_t a,size_t b)602 swap_data(uint64_t *data, size_t a, size_t b)
603 {
604 uint64_t swap = data[a];
605 data[a] = data[b];
606 data[b] = swap;
607 }
608
609 static void
sort_data(uint64_t * data,size_t count)610 sort_data(uint64_t *data, size_t count)
611 {
612 #if 0
613 /*
614 * Mostly classic bubble sort
615 */
616 do {
617 size_t new_count = 0;
618 for (size_t i = 1; i < count; i++) {
619 if (tbs[i - 1] > tbs[i]) {
620 swap_tbs(tbs, i - 1, i);
621 new_count = i;
622 }
623 }
624 count = new_count;
625 } while (count > 0);
626 #else
627 /*
628 * Comb sort
629 */
630 size_t gap = count;
631 bool swapped = false;
632 while (gap > 1 || swapped) {
633 if (gap > 1) {
634 /*
635 * phi = (1 + sqrt(5)) / 2 [golden ratio]
636 * N = 1 / (1 - e^-phi)) = 1.247330950103979
637 *
638 * We want to but can't use floating point to calculate
639 * gap = (size_t)((double)gap / N)
640 *
641 * So we will use the multicative inverse of N
642 * (module 65536) to achieve the division.
643 *
644 * iN = 2^16 / 1.24733... = 52540
645 * x / N == (x * iN) / 65536
646 */
647 gap = (gap * 52540) / 65536;
648 }
649
650 swapped = false;
651
652 for (size_t i = 0; gap + i < count; i++) {
653 if (data[i] > data[i + gap]) {
654 swap_data(data, i, i + gap);
655 swapped = true;
656 }
657 }
658 }
659 #endif
660 }
661 #endif
662
663 void
dump_splhist(struct cpu_info * ci,void (* pr)(const char *,...))664 dump_splhist(struct cpu_info *ci, void (*pr)(const char *, ...))
665 {
666 #ifdef DIAGNOSTIC
667 struct cpu_softc * const cpu = ci->ci_softc;
668 uint64_t tbs[NIPL*NIPL];
669 size_t ntbs = 0;
670 for (size_t to = 0; to < NIPL; to++) {
671 for (size_t from = 0; from < NIPL; from++) {
672 uint64_t tb = cpu->cpu_spl_tb[to][from];
673 if (tb == 0)
674 continue;
675 tbs[ntbs++] = (tb << 8) | (to << 4) | from;
676 }
677 }
678 sort_data(tbs, ntbs);
679
680 if (pr == NULL)
681 pr = printf;
682 uint64_t last_tb = 0;
683 for (size_t i = 0; i < ntbs; i++) {
684 uint64_t tb = tbs[i];
685 size_t from = tb & 15;
686 size_t to = (tb >> 4) & 15;
687 tb >>= 8;
688 (*pr)("%s(%zu) from %zu at %"PRId64"",
689 from < to ? "splraise" : "splx",
690 to, from, tb);
691 if (last_tb && from != IPL_NONE)
692 (*pr)(" (+%"PRId64")", tb - last_tb);
693 (*pr)("\n");
694 last_tb = tb;
695 }
696 #endif
697 }
698