1 /* $NetBSD: octeon_cpunode.c,v 1.22 2022/03/03 06:27:41 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2014 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas of 3am Software Foundry.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31 #define __INTR_PRIVATE
32 #include <sys/cdefs.h>
33
34 __KERNEL_RCSID(0, "$NetBSD: octeon_cpunode.c,v 1.22 2022/03/03 06:27:41 riastradh Exp $");
35
36 #include "locators.h"
37 #include "cpunode.h"
38 #include "opt_multiprocessor.h"
39 #include "opt_ddb.h"
40
41 #include <sys/param.h>
42 #include <sys/atomic.h>
43 #include <sys/cpu.h>
44 #include <sys/device.h>
45 #include <sys/lwp.h>
46 #include <sys/reboot.h>
47 #include <sys/wdog.h>
48
49 #include <uvm/uvm.h>
50
51 #include <dev/sysmon/sysmonvar.h>
52
53 #include <mips/cache.h>
54 #include <mips/mips_opcode.h>
55 #include <mips/mips3_clock.h>
56 #include <mips/mips3_pte.h>
57
58 #include <mips/cavium/octeonvar.h>
59 #include <mips/cavium/dev/octeon_ciureg.h>
60 #include <mips/cavium/dev/octeon_corereg.h>
61
62 extern struct cpu_softc octeon_cpu_softc[];
63
64 struct cpunode_attach_args {
65 const char *cnaa_name;
66 int cnaa_cpunum;
67 };
68
69 struct cpunode_softc {
70 device_t sc_dev;
71 device_t sc_wdog_dev;
72 };
73
74 static int cpunode_mainbus_match(device_t, cfdata_t, void *);
75 static void cpunode_mainbus_attach(device_t, device_t, void *);
76
77 static int cpu_cpunode_match(device_t, cfdata_t, void *);
78 static void cpu_cpunode_attach(device_t, device_t, void *);
79
80 CFATTACH_DECL_NEW(cpunode, sizeof(struct cpunode_softc),
81 cpunode_mainbus_match, cpunode_mainbus_attach, NULL, NULL);
82
83 CFATTACH_DECL_NEW(cpu_cpunode, 0,
84 cpu_cpunode_match, cpu_cpunode_attach, NULL, NULL);
85
86 #ifdef MULTIPROCESSOR
87 CTASSERT(MAXCPUS <= sizeof(uint64_t) * NBBY);
88 volatile uint64_t cpus_booted = __BIT(0); /* cpu0 is always booted */
89 #endif
90
91 static void wdog_cpunode_poke(void *arg);
92
93 static int
cpunode_mainbus_print(void * aux,const char * pnp)94 cpunode_mainbus_print(void *aux, const char *pnp)
95 {
96 struct cpunode_attach_args * const cnaa = aux;
97
98 if (pnp)
99 aprint_normal("%s", pnp);
100
101 if (cnaa->cnaa_cpunum != CPUNODECF_CORE_DEFAULT)
102 aprint_normal(" core %d", cnaa->cnaa_cpunum);
103
104 return UNCONF;
105 }
106
107 int
cpunode_mainbus_match(device_t parent,cfdata_t cf,void * aux)108 cpunode_mainbus_match(device_t parent, cfdata_t cf, void *aux)
109 {
110
111 return 1;
112 }
113
114 void
cpunode_mainbus_attach(device_t parent,device_t self,void * aux)115 cpunode_mainbus_attach(device_t parent, device_t self, void *aux)
116 {
117 struct cpunode_softc * const sc = device_private(self);
118 const uint64_t fuse = octeon_xkphys_read_8(CIU_FUSE);
119 int cpunum = 0;
120
121 sc->sc_dev = self;
122
123 aprint_naive(": %u core%s\n", popcount64(fuse), fuse == 1 ? "" : "s");
124 aprint_normal(": %u core%s", popcount64(fuse), fuse == 1 ? "" : "s");
125
126 const uint64_t cvmctl = mips_cp0_cvmctl_read();
127 aprint_normal(", %scrypto", (cvmctl & CP0_CVMCTL_NOCRYPTO) ? "no " : "");
128 aprint_normal((cvmctl & CP0_CVMCTL_KASUMI) ? "+kasumi" : "");
129 aprint_normal(", %s64bit-mul", (cvmctl & CP0_CVMCTL_NOMUL) ? "no " : "");
130 if (cvmctl & CP0_CVMCTL_REPUN)
131 aprint_normal(", unaligned-access ok");
132 #ifdef MULTIPROCESSOR
133 aprint_normal(", booted %#" PRIx64, cpus_booted);
134 #endif
135 aprint_normal("\n");
136
137 for (uint64_t f = fuse; f != 0; f >>= 1, cpunum++) {
138 struct cpunode_attach_args cnaa = {
139 .cnaa_name = "cpu",
140 .cnaa_cpunum = cpunum,
141 };
142 config_found(self, &cnaa, cpunode_mainbus_print, CFARGS_NONE);
143 }
144 #if NWDOG > 0
145 struct cpunode_attach_args cnaa = {
146 .cnaa_name = "wdog",
147 .cnaa_cpunum = CPUNODECF_CORE_DEFAULT,
148 };
149 config_found(self, &cnaa, cpunode_mainbus_print, CFARGS_NONE);
150 #endif
151 }
152
153 int
cpu_cpunode_match(device_t parent,cfdata_t cf,void * aux)154 cpu_cpunode_match(device_t parent, cfdata_t cf, void *aux)
155 {
156 struct cpunode_attach_args * const cnaa = aux;
157 const int cpunum = cf->cf_loc[CPUNODECF_CORE];
158
159 return strcmp(cnaa->cnaa_name, cf->cf_name) == 0
160 && (cpunum == CPUNODECF_CORE_DEFAULT || cpunum == cnaa->cnaa_cpunum);
161 }
162
163 #if defined(MULTIPROCESSOR)
164 static bool
octeon_fixup_cpu_info_references(int32_t load_addr,uint32_t new_insns[2],void * arg)165 octeon_fixup_cpu_info_references(int32_t load_addr, uint32_t new_insns[2],
166 void *arg)
167 {
168 struct cpu_info * const ci = arg;
169
170 atomic_or_ulong(&curcpu()->ci_flags, CPUF_PRESENT);
171
172 KASSERT(MIPS_KSEG0_P(load_addr));
173 #ifdef MULTIPROCESSOR
174 KASSERT(!CPU_IS_PRIMARY(curcpu()));
175 #endif
176 load_addr += (intptr_t)ci - (intptr_t)&cpu_info_store;
177
178 KASSERT((intptr_t)ci <= load_addr);
179 KASSERT(load_addr < (intptr_t)(ci + 1));
180
181 KASSERT(INSN_LUI_P(new_insns[0]));
182 KASSERT(INSN_LOAD_P(new_insns[1]) || INSN_STORE_P(new_insns[1]));
183
184 /*
185 * Use the lui and load/store instruction as a prototype and
186 * make it refer to cpu1_info_store instead of cpu_info_store.
187 */
188 new_insns[0] &= __BITS(31,16);
189 new_insns[1] &= __BITS(31,16);
190 new_insns[0] |= (uint16_t)((load_addr + 0x8000) >> 16);
191 new_insns[1] |= (uint16_t)load_addr;
192 #ifdef DEBUG_VERBOSE
193 printf("%s: %08x: insn#1 %08x: lui r%u, %d\n",
194 __func__, load_addr, new_insns[0],
195 (new_insns[0] >> 16) & 31,
196 (int16_t)new_insns[0]);
197 printf("%s: %08x: insn#2 %08x: %c%c r%u, %d(r%u)\n",
198 __func__, load_addr, new_insns[1],
199 INSN_LOAD_P(new_insns[1]) ? 'l' : 's',
200 INSN_LW_P(new_insns[1]) ? 'w' : 'd',
201 (new_insns[1] >> 16) & 31,
202 (int16_t)new_insns[1],
203 (new_insns[1] >> 21) & 31);
204 #endif
205 return true;
206 }
207
208 static void
octeon_cpu_init(struct cpu_info * ci)209 octeon_cpu_init(struct cpu_info *ci)
210 {
211 extern const mips_locore_jumpvec_t mips64r2_locore_vec;
212 bool ok __diagused;
213
214 mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
215 mips3_cp0_wired_write(0);
216 (*mips64r2_locore_vec.ljv_tlb_invalidate_all)();
217 mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
218
219 // First thing is setup the exception vectors for this cpu.
220 mips64r2_vector_init(&mips_splsw);
221
222 // Next rewrite those exceptions to use this cpu's cpu_info.
223 ok = mips_fixup_exceptions(octeon_fixup_cpu_info_references, ci);
224 KASSERT(ok);
225
226 (void) splhigh(); // make sure interrupts are masked
227
228 KASSERT((mipsNN_cp0_ebase_read() & MIPS_EBASE_CPUNUM) == ci->ci_cpuid);
229 KASSERT(curcpu() == ci);
230 KASSERT(ci->ci_cpl == IPL_HIGH);
231 KASSERT((mips_cp0_status_read() & MIPS_INT_MASK) == 0);
232 }
233
234 static void
octeon_cpu_run(struct cpu_info * ci)235 octeon_cpu_run(struct cpu_info *ci)
236 {
237
238 octeon_intr_init(ci);
239
240 mips3_initclocks();
241 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl);
242 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
243
244 aprint_normal("%s: ", device_xname(ci->ci_dev));
245 cpu_identify(ci->ci_dev);
246 }
247 #endif /* MULTIPROCESSOR */
248
249 static void
cpu_cpunode_attach_common(device_t self,struct cpu_info * ci)250 cpu_cpunode_attach_common(device_t self, struct cpu_info *ci)
251 {
252 struct cpu_softc * const cpu __diagused = ci->ci_softc;
253
254 ci->ci_dev = self;
255 device_set_private(self, ci);
256
257 KASSERTMSG(cpu != NULL, "ci %p index %d", ci, cpu_index(ci));
258
259 #if NWDOG > 0 || defined(DDB)
260 /* XXXXXX __mips_n32 and MIPS_PHYS_TO_XKPHYS_CACHED needed here?????? */
261 void **nmi_vector = (void *)MIPS_PHYS_TO_KSEG0(0x800 + 32*ci->ci_cpuid);
262 *nmi_vector = octeon_reset_vector;
263
264 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_ZERO);
265 KASSERT(pg != NULL);
266 const vaddr_t kva = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
267 KASSERT(kva != 0);
268 ci->ci_nmi_stack = (void *)(kva + PAGE_SIZE - sizeof(struct kernframe));
269 #endif
270
271 #if NWDOG > 0
272 cpu->cpu_wdog_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE,
273 wdog_cpunode_poke, cpu);
274 KASSERT(cpu->cpu_wdog_sih != NULL);
275 #endif
276
277 aprint_normal(": %lu.%02luMHz\n",
278 (ci->ci_cpu_freq + 5000) / 1000000,
279 ((ci->ci_cpu_freq + 5000) % 1000000) / 10000);
280 aprint_debug_dev(self, "hz cycles = %lu, delay divisor = %lu\n",
281 ci->ci_cycles_per_hz, ci->ci_divisor_delay);
282
283 if (CPU_IS_PRIMARY(ci)) {
284 aprint_normal("%s: ", device_xname(self));
285 cpu_identify(self);
286 }
287 cpu_attach_common(self, ci);
288 #ifdef MULTIPROCESSOR
289 KASSERT(cpuid_infos[ci->ci_cpuid] == ci);
290 #endif
291 }
292
293 void
cpu_cpunode_attach(device_t parent,device_t self,void * aux)294 cpu_cpunode_attach(device_t parent, device_t self, void *aux)
295 {
296 struct cpunode_attach_args * const cnaa = aux;
297 const int cpunum = cnaa->cnaa_cpunum;
298
299 if (cpunum == 0) {
300 cpu_cpunode_attach_common(self, curcpu());
301 #ifdef MULTIPROCESSOR
302 mips_locoresw.lsw_cpu_init = octeon_cpu_init;
303 mips_locoresw.lsw_cpu_run = octeon_cpu_run;
304 #endif
305 return;
306 }
307 #ifdef MULTIPROCESSOR
308 if ((boothowto & RB_MD1) != 0) {
309 aprint_naive("\n");
310 aprint_normal(": multiprocessor boot disabled\n");
311 return;
312 }
313
314 if (!(cpus_booted & __BIT(cpunum))) {
315 aprint_naive(" disabled\n");
316 aprint_normal(" disabled (unresponsive)\n");
317 return;
318 }
319 struct cpu_info * const ci = cpu_info_alloc(NULL, cpunum, 0, cpunum, 0);
320
321 ci->ci_softc = &octeon_cpu_softc[cpunum];
322 ci->ci_softc->cpu_ci = ci;
323
324 cpu_cpunode_attach_common(self, ci);
325
326 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
327 for (int i = 0; i < 100 && !kcpuset_isset(cpus_hatched, cpunum); i++) {
328 delay(10000);
329 }
330 if (!kcpuset_isset(cpus_hatched, cpunum)) {
331 #ifdef DDB
332 aprint_verbose_dev(self, "hatch failed ci=%p flags=%#lx\n", ci, ci->ci_flags);
333 cpu_Debugger();
334 #endif
335 panic("%s failed to hatch: ci=%p flags=%#lx",
336 cpu_name(ci), ci, ci->ci_flags);
337 }
338 #else
339 aprint_naive(": disabled\n");
340 aprint_normal(": disabled (uniprocessor kernel)\n");
341 #endif
342 }
343
344 #if NWDOG > 0
345 struct wdog_softc {
346 struct sysmon_wdog sc_smw;
347 device_t sc_dev;
348 u_int sc_wdog_period;
349 bool sc_wdog_armed;
350 };
351
352 #ifndef OCTEON_WDOG_PERIOD_DEFAULT
353 #define OCTEON_WDOG_PERIOD_DEFAULT 4
354 #endif
355
356 static int wdog_cpunode_match(device_t, cfdata_t, void *);
357 static void wdog_cpunode_attach(device_t, device_t, void *);
358
359 CFATTACH_DECL_NEW(wdog_cpunode, sizeof(struct wdog_softc),
360 wdog_cpunode_match, wdog_cpunode_attach, NULL, NULL);
361
362 static int
wdog_cpunode_setmode(struct sysmon_wdog * smw)363 wdog_cpunode_setmode(struct sysmon_wdog *smw)
364 {
365 struct wdog_softc * const sc = smw->smw_cookie;
366
367 if ((smw->smw_mode & WDOG_MODE_MASK) == WDOG_MODE_DISARMED) {
368 if (sc->sc_wdog_armed) {
369 CPU_INFO_ITERATOR cii;
370 struct cpu_info *ci;
371 for (CPU_INFO_FOREACH(cii, ci)) {
372 struct cpu_softc * const cpu = ci->ci_softc;
373 uint64_t wdog = mips3_ld(cpu->cpu_wdog);
374 wdog &= ~CIU_WDOGX_MODE;
375 mips3_sd(cpu->cpu_pp_poke, wdog);
376 aprint_verbose_dev(sc->sc_dev,
377 "%s: disable wdog=%#"PRIx64"\n",
378 cpu_name(ci), wdog);
379 mips3_sd(cpu->cpu_wdog, wdog);
380 mips3_sd(cpu->cpu_pp_poke, wdog);
381 }
382 sc->sc_wdog_armed = false;
383 }
384 } else if (!sc->sc_wdog_armed) {
385 kpreempt_disable();
386 struct cpu_info *ci = curcpu();
387 if (smw->smw_period == WDOG_PERIOD_DEFAULT) {
388 smw->smw_period = OCTEON_WDOG_PERIOD_DEFAULT;
389 }
390 uint64_t wdog_len = smw->smw_period * ci->ci_cpu_freq;
391 //
392 // This wdog is a 24-bit counter that decrements every 256
393 // cycles. This is then a 32-bit counter so as long wdog_len
394 // doesn't overflow a 32-bit value, we are fine. We write the
395 // 16-bits of the 32-bit period.
396 if ((wdog_len >> 32) != 0) {
397 kpreempt_enable();
398 return EINVAL;
399 }
400 sc->sc_wdog_period = smw->smw_period;
401 CPU_INFO_ITERATOR cii;
402 for (CPU_INFO_FOREACH(cii, ci)) {
403 struct cpu_softc * const cpu = ci->ci_softc;
404 uint64_t wdog = mips3_ld(cpu->cpu_wdog);
405 wdog &= ~(CIU_WDOGX_MODE|CIU_WDOGX_LEN);
406 wdog |= __SHIFTIN(3, CIU_WDOGX_MODE);
407 wdog |= __SHIFTIN(wdog_len >> 16, CIU_WDOGX_LEN);
408 aprint_verbose_dev(sc->sc_dev,
409 "%s: enable wdog=%#"PRIx64" (%#"PRIx64")\n",
410 cpu_name(ci), wdog, wdog_len);
411 mips3_sd(cpu->cpu_wdog, wdog);
412 }
413 sc->sc_wdog_armed = true;
414 kpreempt_enable();
415 }
416 return 0;
417 }
418
419 static void
wdog_cpunode_poke(void * arg)420 wdog_cpunode_poke(void *arg)
421 {
422 struct cpu_softc *cpu = arg;
423
424 mips3_sd(cpu->cpu_pp_poke, 0);
425 }
426
427 static int
wdog_cpunode_tickle(struct sysmon_wdog * smw)428 wdog_cpunode_tickle(struct sysmon_wdog *smw)
429 {
430
431 wdog_cpunode_poke(curcpu()->ci_softc);
432 #ifdef MULTIPROCESSOR
433 // We need to send IPIs to the other CPUs to poke their wdog.
434 cpu_send_ipi(NULL, IPI_WDOG);
435 #endif
436 return 0;
437 }
438
439 int
wdog_cpunode_match(device_t parent,cfdata_t cf,void * aux)440 wdog_cpunode_match(device_t parent, cfdata_t cf, void *aux)
441 {
442 struct cpunode_softc * const sc = device_private(parent);
443 struct cpunode_attach_args * const cnaa = aux;
444 const int cpunum = cf->cf_loc[CPUNODECF_CORE];
445
446 return sc->sc_wdog_dev == NULL
447 && strcmp(cnaa->cnaa_name, cf->cf_name) == 0
448 && cpunum == CPUNODECF_CORE_DEFAULT;
449 }
450
451 void
wdog_cpunode_attach(device_t parent,device_t self,void * aux)452 wdog_cpunode_attach(device_t parent, device_t self, void *aux)
453 {
454 struct cpunode_softc * const psc = device_private(parent);
455 struct wdog_softc * const sc = device_private(self);
456 cfdata_t const cf = device_cfdata(self);
457
458 psc->sc_wdog_dev = self;
459
460 sc->sc_dev = self;
461 sc->sc_smw.smw_name = device_xname(self);
462 sc->sc_smw.smw_cookie = sc;
463 sc->sc_smw.smw_setmode = wdog_cpunode_setmode;
464 sc->sc_smw.smw_tickle = wdog_cpunode_tickle;
465 sc->sc_smw.smw_period = OCTEON_WDOG_PERIOD_DEFAULT;
466 sc->sc_wdog_period = sc->sc_smw.smw_period;
467
468 /*
469 * We need one softint per cpu. It's to tickle the softints on
470 * other CPUs.
471 */
472 #if 0 /* XXX unused? */
473 CPU_INFO_ITERATOR cii;
474 struct cpu_info *ci;
475 for (CPU_INFO_FOREACH(cii, ci)) {
476 }
477 #endif
478
479 aprint_normal(": default period is %u second%s\n",
480 sc->sc_wdog_period, sc->sc_wdog_period == 1 ? "" : "s");
481
482 if (sysmon_wdog_register(&sc->sc_smw) != 0) {
483 aprint_error_dev(self, "unable to register with sysmon\n");
484 return;
485 }
486
487 if (cf->cf_flags & 1) {
488 int error = sysmon_wdog_setmode(&sc->sc_smw, WDOG_MODE_KTICKLE,
489 sc->sc_wdog_period);
490 if (error)
491 aprint_error_dev(self,
492 "failed to start kernel tickler: %d\n", error);
493 }
494 }
495 #endif /* NWDOG > 0 */
496