1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org> 5 * Copyright (c) 2010 Juniper Networks, Inc. 6 * Copyright (c) 2009 Robert N. M. Watson 7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org> 8 * Copyright (c) 2008 Yahoo!, Inc. 9 * All rights reserved. 10 * 11 * Written by: John Baldwin <jhb@FreeBSD.org> 12 * 13 * This software was developed by Robert N. M. Watson under contract 14 * to Juniper Networks, Inc. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. Neither the name of the author nor the names of any co-contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/cdefs.h> 42 #include <sys/param.h> 43 #include <sys/pcpu.h> 44 #include <sys/sysctl.h> 45 #include <kvm.h> 46 #include <limits.h> 47 #include <stdlib.h> 48 49 #include "kvm_private.h" 50 51 #ifdef __amd64__ 52 #define __OFFSET_BY_PCPU 53 #endif 54 55 static struct nlist kvm_pcpu_nl[] = { 56 { .n_name = "_cpuid_to_pcpu" }, 57 { .n_name = "_mp_maxcpus" }, 58 { .n_name = "_mp_ncpus" }, 59 #ifdef __OFFSET_BY_PCPU 60 { .n_name = "___pcpu" }, 61 #endif 62 { .n_name = NULL }, 63 }; 64 #define NL_CPUID_TO_PCPU 0 65 #define NL_MP_MAXCPUS 1 66 #define NL_MP_NCPUS 2 67 #define NL___PCPU 3 68 69 /* 70 * Kernel per-CPU data state. We cache this stuff on the first 71 * access. 72 * 73 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the 74 * consumer has multiple handles in flight to differently configured 75 * kernels/crashdumps. 76 */ 77 static void **pcpu_data; 78 static int maxcpu; 79 static int mp_ncpus; 80 #ifdef __OFFSET_BY_PCPU 81 static unsigned long __pcpu; 82 #endif 83 84 static int 85 _kvm_pcpu_init(kvm_t *kd) 86 { 87 size_t len; 88 int max; 89 void *data; 90 91 if (kvm_nlist(kd, kvm_pcpu_nl) < 0) 92 return (-1); 93 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) { 94 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu"); 95 return (-1); 96 } 97 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) { 98 _kvm_err(kd, kd->program, "unable to find mp_maxcpus"); 99 return (-1); 100 } 101 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max, 102 sizeof(max)) != sizeof(max)) { 103 _kvm_err(kd, kd->program, "cannot read mp_maxcpus"); 104 return (-1); 105 } 106 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) { 107 _kvm_err(kd, kd->program, "unable to find mp_ncpus"); 108 return (-1); 109 } 110 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus, 111 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) { 112 _kvm_err(kd, kd->program, "cannot read mp_ncpus"); 113 return (-1); 114 } 115 #ifdef __OFFSET_BY_PCPU 116 if (kvm_pcpu_nl[NL___PCPU].n_value == 0) { 117 _kvm_err(kd, kd->program, "unable to find __pcpu"); 118 return (-1); 119 } 120 if (kvm_read(kd, kvm_pcpu_nl[NL___PCPU].n_value, &__pcpu, 121 sizeof(__pcpu)) != sizeof(__pcpu)) { 122 _kvm_err(kd, kd->program, "cannot read __pcpu"); 123 return (-1); 124 } 125 #endif 126 len = max * sizeof(void *); 127 data = malloc(len); 128 if (data == NULL) { 129 _kvm_err(kd, kd->program, "out of memory"); 130 return (-1); 131 } 132 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) != 133 (ssize_t)len) { 134 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array"); 135 free(data); 136 return (-1); 137 } 138 pcpu_data = data; 139 maxcpu = max; 140 return (0); 141 } 142 143 static void 144 _kvm_pcpu_clear(void) 145 { 146 147 maxcpu = 0; 148 free(pcpu_data); 149 pcpu_data = NULL; 150 } 151 152 void * 153 kvm_getpcpu(kvm_t *kd, int cpu) 154 { 155 char *buf; 156 157 if (kd == NULL) { 158 _kvm_pcpu_clear(); 159 return (NULL); 160 } 161 162 if (maxcpu == 0) 163 if (_kvm_pcpu_init(kd) < 0) 164 return ((void *)-1); 165 166 if (cpu >= maxcpu || pcpu_data[cpu] == NULL) 167 return (NULL); 168 169 buf = malloc(sizeof(struct pcpu)); 170 if (buf == NULL) { 171 _kvm_err(kd, kd->program, "out of memory"); 172 return ((void *)-1); 173 } 174 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf, 175 sizeof(struct pcpu)) != sizeof(struct pcpu)) { 176 _kvm_err(kd, kd->program, "unable to read per-CPU data"); 177 free(buf); 178 return ((void *)-1); 179 } 180 return (buf); 181 } 182 183 int 184 kvm_getmaxcpu(kvm_t *kd) 185 { 186 187 if (kd == NULL) { 188 _kvm_pcpu_clear(); 189 return (0); 190 } 191 192 if (maxcpu == 0) 193 if (_kvm_pcpu_init(kd) < 0) 194 return (-1); 195 return (maxcpu); 196 } 197 198 int 199 kvm_getncpus(kvm_t *kd) 200 { 201 202 if (mp_ncpus == 0) 203 if (_kvm_pcpu_init(kd) < 0) 204 return (-1); 205 return (mp_ncpus); 206 } 207 208 static int 209 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error) 210 { 211 212 if (!kd->dpcpu_initialized) { 213 if (report_error) 214 _kvm_err(kd, kd->program, "%s: not initialized", 215 __func__); 216 return (-1); 217 } 218 if (cpu >= kd->dpcpu_maxcpus) { 219 if (report_error) 220 _kvm_err(kd, kd->program, "%s: CPU %u too big", 221 __func__, cpu); 222 return (-1); 223 } 224 if (kd->dpcpu_off[cpu] == 0) { 225 if (report_error) 226 _kvm_err(kd, kd->program, "%s: CPU %u not found", 227 __func__, cpu); 228 return (-1); 229 } 230 kd->dpcpu_curcpu = cpu; 231 kd->dpcpu_curoff = kd->dpcpu_off[cpu]; 232 return (0); 233 } 234 235 /* 236 * Set up libkvm to handle dynamic per-CPU memory. 237 */ 238 static int 239 _kvm_dpcpu_init(kvm_t *kd) 240 { 241 struct kvm_nlist nl[] = { 242 #define NLIST_START_SET_PCPU 0 243 { .n_name = "___start_" DPCPU_SETNAME }, 244 #define NLIST_STOP_SET_PCPU 1 245 { .n_name = "___stop_" DPCPU_SETNAME }, 246 #define NLIST_DPCPU_OFF 2 247 { .n_name = "_dpcpu_off" }, 248 #define NLIST_MP_MAXCPUS 3 249 { .n_name = "_mp_maxcpus" }, 250 { .n_name = NULL }, 251 }; 252 uintptr_t *dpcpu_off_buf; 253 size_t len; 254 u_int dpcpu_maxcpus; 255 256 /* 257 * XXX: This only works for native kernels for now. 258 */ 259 if (!kvm_native(kd)) 260 return (-1); 261 262 /* 263 * Locate and cache locations of important symbols using the internal 264 * version of _kvm_nlist, turning off initialization to avoid 265 * recursion in case of unresolveable symbols. 266 */ 267 if (_kvm_nlist(kd, nl, 0) != 0) 268 return (-1); 269 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus, 270 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus)) 271 return (-1); 272 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf); 273 dpcpu_off_buf = malloc(len); 274 if (dpcpu_off_buf == NULL) 275 return (-1); 276 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) != 277 (ssize_t)len) { 278 free(dpcpu_off_buf); 279 return (-1); 280 } 281 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value; 282 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value; 283 kd->dpcpu_maxcpus = dpcpu_maxcpus; 284 kd->dpcpu_off = dpcpu_off_buf; 285 kd->dpcpu_initialized = 1; 286 (void)_kvm_dpcpu_setcpu(kd, 0, 0); 287 return (0); 288 } 289 290 /* 291 * Check whether the dpcpu module has been initialized successfully or not, 292 * initialize it if permitted. 293 */ 294 int 295 _kvm_dpcpu_initialized(kvm_t *kd, int intialize) 296 { 297 298 if (kd->dpcpu_initialized || !intialize) 299 return (kd->dpcpu_initialized); 300 301 (void)_kvm_dpcpu_init(kd); 302 303 return (kd->dpcpu_initialized); 304 } 305 306 /* 307 * Check whether the value is within the dpcpu symbol range and only if so 308 * adjust the offset relative to the current offset. 309 */ 310 kvaddr_t 311 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value) 312 { 313 314 if (value == 0) 315 return (value); 316 317 if (!kd->dpcpu_initialized) 318 return (value); 319 320 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop) 321 return (value); 322 323 return (kd->dpcpu_curoff + value); 324 } 325 326 int 327 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu) 328 { 329 int ret; 330 331 if (!kd->dpcpu_initialized) { 332 ret = _kvm_dpcpu_init(kd); 333 if (ret != 0) { 334 _kvm_err(kd, kd->program, "%s: init failed", 335 __func__); 336 return (ret); 337 } 338 } 339 340 return (_kvm_dpcpu_setcpu(kd, cpu, 1)); 341 } 342 343 /* 344 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation. 345 */ 346 ssize_t 347 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu) 348 { 349 350 if (!kvm_native(kd)) 351 return (-1); 352 if (mp_ncpus == 0) 353 if (_kvm_pcpu_init(kd) < 0) 354 return (0); 355 356 #ifdef __OFFSET_BY_PCPU 357 base += __pcpu; 358 #endif 359 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu), 360 buf, size)); 361 } 362 363 /* 364 * Fetch value of a counter(9). 365 */ 366 uint64_t 367 kvm_counter_u64_fetch(kvm_t *kd, u_long base) 368 { 369 uint64_t r, c; 370 371 if (mp_ncpus == 0) 372 if (_kvm_pcpu_init(kd) < 0) 373 return (0); 374 375 r = 0; 376 for (int i = 0; i < mp_ncpus; i++) { 377 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c)) 378 return (0); 379 r += c; 380 } 381 382 return (r); 383 } 384