1 /*- 2 * Copyright (c) 2001 Wind River Systems, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 4. Neither the name of the author nor the names of any co-contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * This module provides MI support for per-cpu data. 36 * 37 * Each architecture determines the mapping of logical CPU IDs to physical 38 * CPUs. The requirements of this mapping are as follows: 39 * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. 40 * - The mapping is not required to be dense. That is, there may be 41 * gaps in the mappings. 42 * - The platform sets the value of MAXCPU in <machine/param.h>. 43 * - It is suggested, but not required, that in the non-SMP case, the 44 * platform define MAXCPU to be 1 and define the logical ID of the 45 * sole CPU as 0. 46 */ 47 48 #include <sys/cdefs.h> 49 __FBSDID("$FreeBSD$"); 50 51 #include "opt_ddb.h" 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/sysctl.h> 56 #include <sys/linker_set.h> 57 #include <sys/lock.h> 58 #include <sys/malloc.h> 59 #include <sys/pcpu.h> 60 #include <sys/proc.h> 61 #include <sys/smp.h> 62 #include <sys/sx.h> 63 #include <ddb/ddb.h> 64 65 MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); 66 67 struct dpcpu_free { 68 uintptr_t df_start; 69 int df_len; 70 TAILQ_ENTRY(dpcpu_free) df_link; 71 }; 72 73 static DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); 74 static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); 75 static struct sx dpcpu_lock; 76 uintptr_t dpcpu_off[MAXCPU]; 77 struct pcpu *cpuid_to_pcpu[MAXCPU]; 78 struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead); 79 80 /* 81 * Initialize the MI portions of a struct pcpu. 82 */ 83 void 84 pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 85 { 86 87 bzero(pcpu, size); 88 KASSERT(cpuid >= 0 && cpuid < MAXCPU, 89 ("pcpu_init: invalid cpuid %d", cpuid)); 90 pcpu->pc_cpuid = cpuid; 91 pcpu->pc_cpumask = 1 << cpuid; 92 cpuid_to_pcpu[cpuid] = pcpu; 93 SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); 94 cpu_pcpu_init(pcpu, cpuid, size); 95 pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; 96 pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; 97 #ifdef KTR 98 snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid); 99 #endif 100 } 101 102 void 103 dpcpu_init(void *dpcpu, int cpuid) 104 { 105 struct pcpu *pcpu; 106 107 pcpu = pcpu_find(cpuid); 108 pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; 109 110 /* 111 * Initialize defaults from our linker section. 112 */ 113 memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); 114 115 /* 116 * Place it in the global pcpu offset array. 117 */ 118 dpcpu_off[cpuid] = pcpu->pc_dynamic; 119 } 120 121 static void 122 dpcpu_startup(void *dummy __unused) 123 { 124 struct dpcpu_free *df; 125 126 df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 127 df->df_start = (uintptr_t)&DPCPU_NAME(modspace); 128 df->df_len = DPCPU_MODMIN; 129 TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); 130 sx_init(&dpcpu_lock, "dpcpu alloc lock"); 131 } 132 SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); 133 134 /* 135 * First-fit extent based allocator for allocating space in the per-cpu 136 * region reserved for modules. This is only intended for use by the 137 * kernel linkers to place module linker sets. 138 */ 139 void * 140 dpcpu_alloc(int size) 141 { 142 struct dpcpu_free *df; 143 void *s; 144 145 s = NULL; 146 size = roundup2(size, sizeof(void *)); 147 sx_xlock(&dpcpu_lock); 148 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 149 if (df->df_len < size) 150 continue; 151 if (df->df_len == size) { 152 s = (void *)df->df_start; 153 TAILQ_REMOVE(&dpcpu_head, df, df_link); 154 free(df, M_PCPU); 155 break; 156 } 157 s = (void *)df->df_start; 158 df->df_len -= size; 159 df->df_start = df->df_start + size; 160 break; 161 } 162 sx_xunlock(&dpcpu_lock); 163 164 return (s); 165 } 166 167 /* 168 * Free dynamic per-cpu space at module unload time. 169 */ 170 void 171 dpcpu_free(void *s, int size) 172 { 173 struct dpcpu_free *df; 174 struct dpcpu_free *dn; 175 uintptr_t start; 176 uintptr_t end; 177 178 size = roundup2(size, sizeof(void *)); 179 start = (uintptr_t)s; 180 end = start + size; 181 /* 182 * Free a region of space and merge it with as many neighbors as 183 * possible. Keeping the list sorted simplifies this operation. 184 */ 185 sx_xlock(&dpcpu_lock); 186 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 187 if (df->df_start > end) 188 break; 189 /* 190 * If we expand at the end of an entry we may have to 191 * merge it with the one following it as well. 192 */ 193 if (df->df_start + df->df_len == start) { 194 df->df_len += size; 195 dn = TAILQ_NEXT(df, df_link); 196 if (df->df_start + df->df_len == dn->df_start) { 197 df->df_len += dn->df_len; 198 TAILQ_REMOVE(&dpcpu_head, dn, df_link); 199 free(dn, M_PCPU); 200 } 201 sx_xunlock(&dpcpu_lock); 202 return; 203 } 204 if (df->df_start == end) { 205 df->df_start = start; 206 df->df_len += size; 207 sx_xunlock(&dpcpu_lock); 208 return; 209 } 210 } 211 dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 212 dn->df_start = start; 213 dn->df_len = size; 214 if (df) 215 TAILQ_INSERT_BEFORE(df, dn, df_link); 216 else 217 TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); 218 sx_xunlock(&dpcpu_lock); 219 } 220 221 /* 222 * Initialize the per-cpu storage from an updated linker-set region. 223 */ 224 void 225 dpcpu_copy(void *s, int size) 226 { 227 #ifdef SMP 228 uintptr_t dpcpu; 229 int i; 230 231 for (i = 0; i < mp_ncpus; ++i) { 232 dpcpu = dpcpu_off[i]; 233 if (dpcpu == 0) 234 continue; 235 memcpy((void *)(dpcpu + (uintptr_t)s), s, size); 236 } 237 #else 238 memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); 239 #endif 240 } 241 242 /* 243 * Destroy a struct pcpu. 244 */ 245 void 246 pcpu_destroy(struct pcpu *pcpu) 247 { 248 249 SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); 250 cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; 251 dpcpu_off[pcpu->pc_cpuid] = 0; 252 } 253 254 /* 255 * Locate a struct pcpu by cpu id. 256 */ 257 struct pcpu * 258 pcpu_find(u_int cpuid) 259 { 260 261 return (cpuid_to_pcpu[cpuid]); 262 } 263 264 int 265 sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) 266 { 267 uintptr_t dpcpu; 268 int64_t count; 269 int i; 270 271 count = 0; 272 for (i = 0; i < mp_ncpus; ++i) { 273 dpcpu = dpcpu_off[i]; 274 if (dpcpu == 0) 275 continue; 276 count += *(int64_t *)(dpcpu + (uintptr_t)arg1); 277 } 278 return (SYSCTL_OUT(req, &count, sizeof(count))); 279 } 280 281 int 282 sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) 283 { 284 uintptr_t dpcpu; 285 long count; 286 int i; 287 288 count = 0; 289 for (i = 0; i < mp_ncpus; ++i) { 290 dpcpu = dpcpu_off[i]; 291 if (dpcpu == 0) 292 continue; 293 count += *(long *)(dpcpu + (uintptr_t)arg1); 294 } 295 return (SYSCTL_OUT(req, &count, sizeof(count))); 296 } 297 298 int 299 sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) 300 { 301 uintptr_t dpcpu; 302 int count; 303 int i; 304 305 count = 0; 306 for (i = 0; i < mp_ncpus; ++i) { 307 dpcpu = dpcpu_off[i]; 308 if (dpcpu == 0) 309 continue; 310 count += *(int *)(dpcpu + (uintptr_t)arg1); 311 } 312 return (SYSCTL_OUT(req, &count, sizeof(count))); 313 } 314 315 #ifdef DDB 316 DB_SHOW_COMMAND(dpcpu_off, db_show_dpcpu_off) 317 { 318 int id; 319 320 CPU_FOREACH(id) { 321 db_printf("dpcpu_off[%2d] = 0x%jx (+ DPCPU_START = %p)\n", 322 id, (uintmax_t)dpcpu_off[id], 323 (void *)(uintptr_t)(dpcpu_off[id] + DPCPU_START)); 324 } 325 } 326 327 static void 328 show_pcpu(struct pcpu *pc) 329 { 330 struct thread *td; 331 332 db_printf("cpuid = %d\n", pc->pc_cpuid); 333 db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); 334 db_printf("curthread = "); 335 td = pc->pc_curthread; 336 if (td != NULL) 337 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 338 td->td_name); 339 else 340 db_printf("none\n"); 341 db_printf("curpcb = %p\n", pc->pc_curpcb); 342 db_printf("fpcurthread = "); 343 td = pc->pc_fpcurthread; 344 if (td != NULL) 345 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 346 td->td_name); 347 else 348 db_printf("none\n"); 349 db_printf("idlethread = "); 350 td = pc->pc_idlethread; 351 if (td != NULL) 352 db_printf("%p: tid %d \"%s\"\n", td, td->td_tid, td->td_name); 353 else 354 db_printf("none\n"); 355 db_show_mdpcpu(pc); 356 357 #ifdef VIMAGE 358 db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); 359 #endif 360 361 #ifdef WITNESS 362 db_printf("spin locks held:\n"); 363 witness_list_locks(&pc->pc_spinlocks, db_printf); 364 #endif 365 } 366 367 DB_SHOW_COMMAND(pcpu, db_show_pcpu) 368 { 369 struct pcpu *pc; 370 int id; 371 372 if (have_addr) 373 id = ((addr >> 4) % 16) * 10 + (addr % 16); 374 else 375 id = PCPU_GET(cpuid); 376 pc = pcpu_find(id); 377 if (pc == NULL) { 378 db_printf("CPU %d not found\n", id); 379 return; 380 } 381 show_pcpu(pc); 382 } 383 384 DB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) 385 { 386 struct pcpu *pc; 387 int id; 388 389 db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); 390 for (id = 0; id <= mp_maxid; id++) { 391 pc = pcpu_find(id); 392 if (pc != NULL) { 393 show_pcpu(pc); 394 db_printf("\n"); 395 } 396 } 397 } 398 DB_SHOW_ALIAS(allpcpu, db_show_cpu_all); 399 #endif 400