1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org>
5 * Copyright (c) 2010 Juniper Networks, Inc.
6 * Copyright (c) 2009 Robert N. M. Watson
7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org>
8 * Copyright (c) 2008 Yahoo!, Inc.
9 * All rights reserved.
10 *
11 * Written by: John Baldwin <jhb@FreeBSD.org>
12 *
13 * This software was developed by Robert N. M. Watson under contract
14 * to Juniper Networks, Inc.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. Neither the name of the author nor the names of any co-contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41 #include <sys/param.h>
42 #include <sys/pcpu.h>
43 #include <sys/sysctl.h>
44 #include <kvm.h>
45 #include <limits.h>
46 #include <stdlib.h>
47
48 #include "kvm_private.h"
49
50 #ifdef __amd64__
51 #define __OFFSET_BY_PCPU
52 #endif
53
54 static struct nlist kvm_pcpu_nl[] = {
55 { .n_name = "_cpuid_to_pcpu" },
56 { .n_name = "_mp_maxcpus" },
57 { .n_name = "_mp_ncpus" },
58 #ifdef __OFFSET_BY_PCPU
59 { .n_name = "___pcpu" },
60 #endif
61 { .n_name = NULL },
62 };
63 #define NL_CPUID_TO_PCPU 0
64 #define NL_MP_MAXCPUS 1
65 #define NL_MP_NCPUS 2
66 #define NL___PCPU 3
67
68 /*
69 * Kernel per-CPU data state. We cache this stuff on the first
70 * access.
71 *
72 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
73 * consumer has multiple handles in flight to differently configured
74 * kernels/crashdumps.
75 */
76 static void **pcpu_data;
77 static int maxcpu;
78 static int mp_ncpus;
79 #ifdef __OFFSET_BY_PCPU
80 static unsigned long __pcpu;
81 #endif
82
83 static int
_kvm_pcpu_init(kvm_t * kd)84 _kvm_pcpu_init(kvm_t *kd)
85 {
86 size_t len;
87 int max;
88 void *data;
89
90 if (kvm_nlist(kd, kvm_pcpu_nl) < 0)
91 return (-1);
92 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) {
93 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu");
94 return (-1);
95 }
96 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) {
97 _kvm_err(kd, kd->program, "unable to find mp_maxcpus");
98 return (-1);
99 }
100 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max,
101 sizeof(max)) != sizeof(max)) {
102 _kvm_err(kd, kd->program, "cannot read mp_maxcpus");
103 return (-1);
104 }
105 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) {
106 _kvm_err(kd, kd->program, "unable to find mp_ncpus");
107 return (-1);
108 }
109 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus,
110 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) {
111 _kvm_err(kd, kd->program, "cannot read mp_ncpus");
112 return (-1);
113 }
114 #ifdef __OFFSET_BY_PCPU
115 if (kvm_pcpu_nl[NL___PCPU].n_value == 0) {
116 _kvm_err(kd, kd->program, "unable to find __pcpu");
117 return (-1);
118 }
119 if (kvm_read(kd, kvm_pcpu_nl[NL___PCPU].n_value, &__pcpu,
120 sizeof(__pcpu)) != sizeof(__pcpu)) {
121 _kvm_err(kd, kd->program, "cannot read __pcpu");
122 return (-1);
123 }
124 #endif
125 len = max * sizeof(void *);
126 data = malloc(len);
127 if (data == NULL) {
128 _kvm_err(kd, kd->program, "out of memory");
129 return (-1);
130 }
131 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) !=
132 (ssize_t)len) {
133 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array");
134 free(data);
135 return (-1);
136 }
137 pcpu_data = data;
138 maxcpu = max;
139 return (0);
140 }
141
142 static void
_kvm_pcpu_clear(void)143 _kvm_pcpu_clear(void)
144 {
145
146 maxcpu = 0;
147 free(pcpu_data);
148 pcpu_data = NULL;
149 }
150
151 void *
kvm_getpcpu(kvm_t * kd,int cpu)152 kvm_getpcpu(kvm_t *kd, int cpu)
153 {
154 char *buf;
155
156 if (kd == NULL) {
157 _kvm_pcpu_clear();
158 return (NULL);
159 }
160
161 if (maxcpu == 0)
162 if (_kvm_pcpu_init(kd) < 0)
163 return ((void *)-1);
164
165 if (cpu >= maxcpu || pcpu_data[cpu] == NULL)
166 return (NULL);
167
168 buf = malloc(sizeof(struct pcpu));
169 if (buf == NULL) {
170 _kvm_err(kd, kd->program, "out of memory");
171 return ((void *)-1);
172 }
173 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
174 sizeof(struct pcpu)) != sizeof(struct pcpu)) {
175 _kvm_err(kd, kd->program, "unable to read per-CPU data");
176 free(buf);
177 return ((void *)-1);
178 }
179 return (buf);
180 }
181
182 int
kvm_getmaxcpu(kvm_t * kd)183 kvm_getmaxcpu(kvm_t *kd)
184 {
185
186 if (kd == NULL) {
187 _kvm_pcpu_clear();
188 return (0);
189 }
190
191 if (maxcpu == 0)
192 if (_kvm_pcpu_init(kd) < 0)
193 return (-1);
194 return (maxcpu);
195 }
196
197 int
kvm_getncpus(kvm_t * kd)198 kvm_getncpus(kvm_t *kd)
199 {
200
201 if (mp_ncpus == 0)
202 if (_kvm_pcpu_init(kd) < 0)
203 return (-1);
204 return (mp_ncpus);
205 }
206
207 static int
_kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu,int report_error)208 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
209 {
210
211 if (!kd->dpcpu_initialized) {
212 if (report_error)
213 _kvm_err(kd, kd->program, "%s: not initialized",
214 __func__);
215 return (-1);
216 }
217 if (cpu >= kd->dpcpu_maxcpus) {
218 if (report_error)
219 _kvm_err(kd, kd->program, "%s: CPU %u too big",
220 __func__, cpu);
221 return (-1);
222 }
223 if (kd->dpcpu_off[cpu] == 0) {
224 if (report_error)
225 _kvm_err(kd, kd->program, "%s: CPU %u not found",
226 __func__, cpu);
227 return (-1);
228 }
229 kd->dpcpu_curcpu = cpu;
230 kd->dpcpu_curoff = kd->dpcpu_off[cpu];
231 return (0);
232 }
233
234 /*
235 * Set up libkvm to handle dynamic per-CPU memory.
236 */
237 static int
_kvm_dpcpu_init(kvm_t * kd)238 _kvm_dpcpu_init(kvm_t *kd)
239 {
240 struct kvm_nlist nl[] = {
241 #define NLIST_START_SET_PCPU 0
242 { .n_name = "___start_" DPCPU_SETNAME },
243 #define NLIST_STOP_SET_PCPU 1
244 { .n_name = "___stop_" DPCPU_SETNAME },
245 #define NLIST_DPCPU_OFF 2
246 { .n_name = "_dpcpu_off" },
247 #define NLIST_MP_MAXCPUS 3
248 { .n_name = "_mp_maxcpus" },
249 { .n_name = NULL },
250 };
251 uintptr_t *dpcpu_off_buf;
252 size_t len;
253 u_int dpcpu_maxcpus;
254
255 /*
256 * XXX: This only works for native kernels for now.
257 */
258 if (!kvm_native(kd))
259 return (-1);
260
261 /*
262 * Locate and cache locations of important symbols using the internal
263 * version of _kvm_nlist, turning off initialization to avoid
264 * recursion in case of unresolveable symbols.
265 */
266 if (_kvm_nlist(kd, nl, 0) != 0)
267 return (-1);
268 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
269 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
270 return (-1);
271 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
272 dpcpu_off_buf = malloc(len);
273 if (dpcpu_off_buf == NULL)
274 return (-1);
275 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
276 (ssize_t)len) {
277 free(dpcpu_off_buf);
278 return (-1);
279 }
280 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
281 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
282 kd->dpcpu_maxcpus = dpcpu_maxcpus;
283 kd->dpcpu_off = dpcpu_off_buf;
284 kd->dpcpu_initialized = 1;
285 (void)_kvm_dpcpu_setcpu(kd, 0, 0);
286 return (0);
287 }
288
289 /*
290 * Check whether the dpcpu module has been initialized successfully or not,
291 * initialize it if permitted.
292 */
293 int
_kvm_dpcpu_initialized(kvm_t * kd,int intialize)294 _kvm_dpcpu_initialized(kvm_t *kd, int intialize)
295 {
296
297 if (kd->dpcpu_initialized || !intialize)
298 return (kd->dpcpu_initialized);
299
300 (void)_kvm_dpcpu_init(kd);
301
302 return (kd->dpcpu_initialized);
303 }
304
305 /*
306 * Check whether the value is within the dpcpu symbol range and only if so
307 * adjust the offset relative to the current offset.
308 */
309 kvaddr_t
_kvm_dpcpu_validaddr(kvm_t * kd,kvaddr_t value)310 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value)
311 {
312
313 if (value == 0)
314 return (value);
315
316 if (!kd->dpcpu_initialized)
317 return (value);
318
319 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
320 return (value);
321
322 return (kd->dpcpu_curoff + value);
323 }
324
325 int
kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu)326 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
327 {
328 int ret;
329
330 if (!kd->dpcpu_initialized) {
331 ret = _kvm_dpcpu_init(kd);
332 if (ret != 0) {
333 _kvm_err(kd, kd->program, "%s: init failed",
334 __func__);
335 return (ret);
336 }
337 }
338
339 return (_kvm_dpcpu_setcpu(kd, cpu, 1));
340 }
341
342 /*
343 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation.
344 */
345 ssize_t
kvm_read_zpcpu(kvm_t * kd,u_long base,void * buf,size_t size,int cpu)346 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu)
347 {
348
349 if (!kvm_native(kd))
350 return (-1);
351 if (mp_ncpus == 0)
352 if (_kvm_pcpu_init(kd) < 0)
353 return (0);
354
355 #ifdef __OFFSET_BY_PCPU
356 base += __pcpu;
357 #endif
358 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu),
359 buf, size));
360 }
361
362 /*
363 * Fetch value of a counter(9).
364 */
365 uint64_t
kvm_counter_u64_fetch(kvm_t * kd,u_long base)366 kvm_counter_u64_fetch(kvm_t *kd, u_long base)
367 {
368 uint64_t r, c;
369
370 if (mp_ncpus == 0)
371 if (_kvm_pcpu_init(kd) < 0)
372 return (0);
373
374 r = 0;
375 for (int i = 0; i < mp_ncpus; i++) {
376 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c))
377 return (0);
378 r += c;
379 }
380
381 return (r);
382 }
383