1 /* $NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*-
33 * Copyright (c)2007 YAMAMOTO Takashi,
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 *
45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * SUCH DAMAGE.
56 */
57
58 /*
59 * CPU related routines not shared with rump.
60 */
61
62 #include <sys/cdefs.h>
63 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $");
64
65 #ifdef _KERNEL_OPT
66 #include "opt_cpu_ucode.h"
67 #include "opt_heartbeat.h"
68 #endif
69
70 #include <sys/param.h>
71 #include <sys/systm.h>
72 #include <sys/idle.h>
73 #include <sys/sched.h>
74 #include <sys/intr.h>
75 #include <sys/conf.h>
76 #include <sys/cpu.h>
77 #include <sys/cpuio.h>
78 #include <sys/proc.h>
79 #include <sys/percpu.h>
80 #include <sys/kernel.h>
81 #include <sys/kauth.h>
82 #include <sys/xcall.h>
83 #include <sys/pool.h>
84 #include <sys/kmem.h>
85 #include <sys/select.h>
86 #include <sys/namei.h>
87 #include <sys/callout.h>
88 #include <sys/pcu.h>
89 #include <sys/heartbeat.h>
90
91 #include <uvm/uvm_extern.h>
92
93 #include "ioconf.h"
94
95 /*
96 * If the port has stated that cpu_data is the first thing in cpu_info,
97 * verify that the claim is true. This will prevent them from getting out
98 * of sync.
99 */
100 #ifdef __HAVE_CPU_DATA_FIRST
101 CTASSERT(offsetof(struct cpu_info, ci_data) == 0);
102 #else
103 CTASSERT(offsetof(struct cpu_info, ci_data) != 0);
104 #endif
105
106 int (*compat_cpuctl_ioctl)(struct lwp *, u_long, void *) = (void *)enosys;
107
108 static void cpu_xc_online(struct cpu_info *, void *);
109 static void cpu_xc_offline(struct cpu_info *, void *);
110
111 dev_type_ioctl(cpuctl_ioctl);
112
113 const struct cdevsw cpuctl_cdevsw = {
114 .d_open = nullopen,
115 .d_close = nullclose,
116 .d_read = nullread,
117 .d_write = nullwrite,
118 .d_ioctl = cpuctl_ioctl,
119 .d_stop = nullstop,
120 .d_tty = notty,
121 .d_poll = nopoll,
122 .d_mmap = nommap,
123 .d_kqfilter = nokqfilter,
124 .d_discard = nodiscard,
125 .d_flag = D_OTHER | D_MPSAFE
126 };
127
128 int
mi_cpu_attach(struct cpu_info * ci)129 mi_cpu_attach(struct cpu_info *ci)
130 {
131 int error;
132
133 KASSERT(maxcpus > 0);
134
135 if ((ci->ci_index = ncpu) >= maxcpus)
136 panic("Too many CPUs. Increase MAXCPUS?");
137 kcpuset_set(kcpuset_attached, cpu_index(ci));
138
139 /*
140 * Create a convenience cpuset of just ourselves.
141 */
142 kcpuset_create(&ci->ci_kcpuset, true);
143 kcpuset_set(ci->ci_kcpuset, cpu_index(ci));
144
145 TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
146 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
147
148 /* This is useful for eg, per-cpu evcnt */
149 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d",
150 cpu_index(ci));
151
152 if (__predict_false(cpu_infos == NULL)) {
153 size_t ci_bufsize = (maxcpus + 1) * sizeof(struct cpu_info *);
154 cpu_infos = kmem_zalloc(ci_bufsize, KM_SLEEP);
155 }
156 cpu_infos[cpu_index(ci)] = ci;
157
158 sched_cpuattach(ci);
159
160 error = create_idle_lwp(ci);
161 if (error != 0) {
162 /* XXX revert sched_cpuattach */
163 return error;
164 }
165
166 if (ci == curcpu())
167 ci->ci_onproc = curlwp;
168 else
169 ci->ci_onproc = ci->ci_data.cpu_idlelwp;
170
171 percpu_init_cpu(ci);
172 softint_init(ci);
173 callout_init_cpu(ci);
174 xc_init_cpu(ci);
175 pool_cache_cpu_init(ci);
176 selsysinit(ci);
177 cache_cpu_init(ci);
178 TAILQ_INIT(&ci->ci_data.cpu_biodone);
179 ncpu++;
180 ncpuonline++;
181
182 return 0;
183 }
184
185 void
cpuctlattach(int dummy __unused)186 cpuctlattach(int dummy __unused)
187 {
188
189 KASSERT(cpu_infos != NULL);
190 }
191
192 int
cpuctl_ioctl(dev_t dev,u_long cmd,void * data,int flag,lwp_t * l)193 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
194 {
195 CPU_INFO_ITERATOR cii;
196 cpustate_t *cs;
197 struct cpu_info *ci;
198 int error, i;
199 u_int id;
200
201 error = 0;
202
203 mutex_enter(&cpu_lock);
204 switch (cmd) {
205 case IOC_CPU_SETSTATE:
206 cs = data;
207 error = kauth_authorize_system(l->l_cred,
208 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL,
209 NULL);
210 if (error != 0)
211 break;
212 if (cs->cs_id >= maxcpus ||
213 (ci = cpu_lookup(cs->cs_id)) == NULL) {
214 error = ESRCH;
215 break;
216 }
217 cpu_setintr(ci, cs->cs_intr); /* XXX neglect errors */
218 error = cpu_setstate(ci, cs->cs_online);
219 break;
220
221 case IOC_CPU_GETSTATE:
222 cs = data;
223 id = cs->cs_id;
224 memset(cs, 0, sizeof(*cs));
225 cs->cs_id = id;
226 if (cs->cs_id >= maxcpus ||
227 (ci = cpu_lookup(id)) == NULL) {
228 error = ESRCH;
229 break;
230 }
231 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
232 cs->cs_online = false;
233 else
234 cs->cs_online = true;
235 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
236 cs->cs_intr = false;
237 else
238 cs->cs_intr = true;
239 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod;
240 cs->cs_lastmodhi = (int32_t)
241 (ci->ci_schedstate.spc_lastmod >> 32);
242 cs->cs_intrcnt = cpu_intr_count(ci) + 1;
243 cs->cs_hwid = ci->ci_cpuid;
244 break;
245
246 case IOC_CPU_MAPID:
247 i = 0;
248 for (CPU_INFO_FOREACH(cii, ci)) {
249 if (i++ == *(int *)data)
250 break;
251 }
252 if (ci == NULL)
253 error = ESRCH;
254 else
255 *(int *)data = cpu_index(ci);
256 break;
257
258 case IOC_CPU_GETCOUNT:
259 *(int *)data = ncpu;
260 break;
261
262 #ifdef CPU_UCODE
263 case IOC_CPU_UCODE_GET_VERSION:
264 error = cpu_ucode_get_version((struct cpu_ucode_version *)data);
265 break;
266
267 case IOC_CPU_UCODE_APPLY:
268 error = kauth_authorize_machdep(l->l_cred,
269 KAUTH_MACHDEP_CPU_UCODE_APPLY,
270 NULL, NULL, NULL, NULL);
271 if (error != 0)
272 break;
273 error = cpu_ucode_apply((const struct cpu_ucode *)data);
274 break;
275 #endif
276
277 default:
278 error = (*compat_cpuctl_ioctl)(l, cmd, data);
279 break;
280 }
281 mutex_exit(&cpu_lock);
282
283 return error;
284 }
285
286 struct cpu_info *
cpu_lookup(u_int idx)287 cpu_lookup(u_int idx)
288 {
289 struct cpu_info *ci;
290
291 /*
292 * cpu_infos is a NULL terminated array of MAXCPUS + 1 entries,
293 * so an index of MAXCPUS here is ok. See mi_cpu_attach.
294 */
295 KASSERT(idx <= maxcpus);
296
297 if (__predict_false(cpu_infos == NULL)) {
298 KASSERT(idx == 0);
299 return curcpu();
300 }
301
302 ci = cpu_infos[idx];
303 KASSERT(ci == NULL || cpu_index(ci) == idx);
304 KASSERTMSG(idx < maxcpus || ci == NULL, "idx %d ci %p", idx, ci);
305
306 return ci;
307 }
308
309 static void
cpu_xc_offline(struct cpu_info * ci,void * unused)310 cpu_xc_offline(struct cpu_info *ci, void *unused)
311 {
312 struct schedstate_percpu *spc, *mspc = NULL;
313 struct cpu_info *target_ci;
314 struct lwp *l;
315 CPU_INFO_ITERATOR cii;
316 int s;
317
318 /*
319 * Thread that made the cross call (separate context) holds
320 * cpu_lock on our behalf.
321 */
322 spc = &ci->ci_schedstate;
323 s = splsched();
324 spc->spc_flags |= SPCF_OFFLINE;
325 splx(s);
326
327 /* Take the first available CPU for the migration. */
328 for (CPU_INFO_FOREACH(cii, target_ci)) {
329 mspc = &target_ci->ci_schedstate;
330 if ((mspc->spc_flags & SPCF_OFFLINE) == 0)
331 break;
332 }
333 KASSERT(target_ci != NULL);
334
335 /*
336 * Migrate all non-bound threads to the other CPU. Note that this
337 * runs from the xcall thread, thus handling of LSONPROC is not needed.
338 */
339 mutex_enter(&proc_lock);
340 LIST_FOREACH(l, &alllwp, l_list) {
341 struct cpu_info *mci;
342
343 lwp_lock(l);
344 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) {
345 lwp_unlock(l);
346 continue;
347 }
348 /* Regular case - no affinity. */
349 if (l->l_affinity == NULL) {
350 lwp_migrate(l, target_ci);
351 continue;
352 }
353 /* Affinity is set, find an online CPU in the set. */
354 for (CPU_INFO_FOREACH(cii, mci)) {
355 mspc = &mci->ci_schedstate;
356 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
357 kcpuset_isset(l->l_affinity, cpu_index(mci)))
358 break;
359 }
360 if (mci == NULL) {
361 lwp_unlock(l);
362 mutex_exit(&proc_lock);
363 goto fail;
364 }
365 lwp_migrate(l, mci);
366 }
367 mutex_exit(&proc_lock);
368
369 #if PCU_UNIT_COUNT > 0
370 pcu_save_all_on_cpu();
371 #endif
372
373 #ifdef __HAVE_MD_CPU_OFFLINE
374 cpu_offline_md();
375 #endif
376 return;
377 fail:
378 /* Just unset the SPCF_OFFLINE flag, caller will check */
379 s = splsched();
380 spc->spc_flags &= ~SPCF_OFFLINE;
381 splx(s);
382
383 #ifdef HEARTBEAT
384 heartbeat_suspend();
385 #endif
386 }
387
388 static void
cpu_xc_online(struct cpu_info * ci,void * unused)389 cpu_xc_online(struct cpu_info *ci, void *unused)
390 {
391 struct schedstate_percpu *spc;
392 int s;
393
394 #ifdef HEARTBEAT
395 heartbeat_resume();
396 #endif
397
398 spc = &ci->ci_schedstate;
399 s = splsched();
400 spc->spc_flags &= ~SPCF_OFFLINE;
401 splx(s);
402 }
403
404 int
cpu_setstate(struct cpu_info * ci,bool online)405 cpu_setstate(struct cpu_info *ci, bool online)
406 {
407 struct schedstate_percpu *spc;
408 CPU_INFO_ITERATOR cii;
409 struct cpu_info *ci2;
410 uint64_t where;
411 xcfunc_t func;
412 int nonline;
413
414 spc = &ci->ci_schedstate;
415
416 KASSERT(mutex_owned(&cpu_lock));
417
418 if (online) {
419 if ((spc->spc_flags & SPCF_OFFLINE) == 0)
420 return 0;
421 func = (xcfunc_t)cpu_xc_online;
422 } else {
423 if ((spc->spc_flags & SPCF_OFFLINE) != 0)
424 return 0;
425 nonline = 0;
426 /*
427 * Ensure that at least one CPU within the processor set
428 * stays online. Revisit this later.
429 */
430 for (CPU_INFO_FOREACH(cii, ci2)) {
431 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
432 continue;
433 if (ci2->ci_schedstate.spc_psid != spc->spc_psid)
434 continue;
435 nonline++;
436 }
437 if (nonline == 1)
438 return EBUSY;
439 func = (xcfunc_t)cpu_xc_offline;
440 }
441
442 where = xc_unicast(0, func, ci, NULL, ci);
443 xc_wait(where);
444 if (online) {
445 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0);
446 ncpuonline++;
447 } else {
448 if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
449 /* If was not set offline, then it is busy */
450 return EBUSY;
451 }
452 ncpuonline--;
453 }
454
455 spc->spc_lastmod = time_second;
456 return 0;
457 }
458
459 #if defined(__HAVE_INTR_CONTROL)
460 static void
cpu_xc_intr(struct cpu_info * ci,void * unused)461 cpu_xc_intr(struct cpu_info *ci, void *unused)
462 {
463 struct schedstate_percpu *spc;
464 int s;
465
466 spc = &ci->ci_schedstate;
467 s = splsched();
468 spc->spc_flags &= ~SPCF_NOINTR;
469 splx(s);
470 }
471
472 static void
cpu_xc_nointr(struct cpu_info * ci,void * unused)473 cpu_xc_nointr(struct cpu_info *ci, void *unused)
474 {
475 struct schedstate_percpu *spc;
476 int s;
477
478 spc = &ci->ci_schedstate;
479 s = splsched();
480 spc->spc_flags |= SPCF_NOINTR;
481 splx(s);
482 }
483
484 int
cpu_setintr(struct cpu_info * ci,bool intr)485 cpu_setintr(struct cpu_info *ci, bool intr)
486 {
487 struct schedstate_percpu *spc;
488 CPU_INFO_ITERATOR cii;
489 struct cpu_info *ci2;
490 uint64_t where;
491 xcfunc_t func;
492 int nintr;
493
494 spc = &ci->ci_schedstate;
495
496 KASSERT(mutex_owned(&cpu_lock));
497
498 if (intr) {
499 if ((spc->spc_flags & SPCF_NOINTR) == 0)
500 return 0;
501 func = (xcfunc_t)cpu_xc_intr;
502 } else {
503 if (CPU_IS_PRIMARY(ci)) /* XXX kern/45117 */
504 return EINVAL;
505 if ((spc->spc_flags & SPCF_NOINTR) != 0)
506 return 0;
507 /*
508 * Ensure that at least one CPU within the system
509 * is handing device interrupts.
510 */
511 nintr = 0;
512 for (CPU_INFO_FOREACH(cii, ci2)) {
513 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
514 continue;
515 if (ci2 == ci)
516 continue;
517 nintr++;
518 }
519 if (nintr == 0)
520 return EBUSY;
521 func = (xcfunc_t)cpu_xc_nointr;
522 }
523
524 where = xc_unicast(0, func, ci, NULL, ci);
525 xc_wait(where);
526 if (intr) {
527 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0);
528 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) {
529 /* If was not set offline, then it is busy */
530 return EBUSY;
531 }
532
533 /* Direct interrupts away from the CPU and record the change. */
534 cpu_intr_redistribute();
535 spc->spc_lastmod = time_second;
536 return 0;
537 }
538 #else /* __HAVE_INTR_CONTROL */
539 int
cpu_setintr(struct cpu_info * ci,bool intr)540 cpu_setintr(struct cpu_info *ci, bool intr)
541 {
542
543 return EOPNOTSUPP;
544 }
545
546 u_int
cpu_intr_count(struct cpu_info * ci)547 cpu_intr_count(struct cpu_info *ci)
548 {
549
550 return 0; /* 0 == "don't know" */
551 }
552 #endif /* __HAVE_INTR_CONTROL */
553
554 #ifdef CPU_UCODE
555 int
cpu_ucode_load(struct cpu_ucode_softc * sc,const char * fwname)556 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname)
557 {
558 firmware_handle_t fwh;
559 int error;
560
561 if (sc->sc_blob != NULL) {
562 firmware_free(sc->sc_blob, sc->sc_blobsize);
563 sc->sc_blob = NULL;
564 sc->sc_blobsize = 0;
565 }
566
567 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname);
568 if (error != 0) {
569 #ifdef DEBUG
570 printf("ucode: firmware_open(%s) failed: %i\n", fwname, error);
571 #endif
572 goto err0;
573 }
574
575 sc->sc_blobsize = firmware_get_size(fwh);
576 if (sc->sc_blobsize == 0) {
577 error = EFTYPE;
578 firmware_close(fwh);
579 goto err0;
580 }
581 sc->sc_blob = firmware_malloc(sc->sc_blobsize);
582 if (sc->sc_blob == NULL) {
583 error = ENOMEM;
584 firmware_close(fwh);
585 goto err0;
586 }
587
588 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize);
589 firmware_close(fwh);
590 if (error != 0)
591 goto err1;
592
593 return 0;
594
595 err1:
596 firmware_free(sc->sc_blob, sc->sc_blobsize);
597 sc->sc_blob = NULL;
598 sc->sc_blobsize = 0;
599 err0:
600 return error;
601 }
602 #endif
603