xref: /freebsd/lib/libmemstat/memstat_malloc.c (revision c1d255d3) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2005 Robert N. M. Watson
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/malloc.h>
34 #include <sys/sysctl.h>
35 
36 #include <err.h>
37 #include <errno.h>
38 #include <kvm.h>
39 #include <nlist.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43 
44 #include "memstat.h"
45 #include "memstat_internal.h"
46 
47 static int memstat_malloc_zone_count;
48 static int memstat_malloc_zone_sizes[32];
49 
50 static int	memstat_malloc_zone_init(void);
51 static int	memstat_malloc_zone_init_kvm(kvm_t *kvm);
52 
53 static struct nlist namelist[] = {
54 #define	X_KMEMSTATISTICS	0
55 	{ .n_name = "_kmemstatistics" },
56 #define	X_KMEMZONES		1
57 	{ .n_name = "_kmemzones" },
58 #define	X_NUMZONES		2
59 	{ .n_name = "_numzones" },
60 #define	X_VM_MALLOC_ZONE_COUNT	3
61 	{ .n_name = "_vm_malloc_zone_count" },
62 #define	X_MP_MAXCPUS		4
63 	{ .n_name = "_mp_maxcpus" },
64 	{ .n_name = "" },
65 };
66 
67 /*
68  * Extract malloc(9) statistics from the running kernel, and store all memory
69  * type information in the passed list.  For each type, check the list for an
70  * existing entry with the right name/allocator -- if present, update that
71  * entry.  Otherwise, add a new entry.  On error, the entire list will be
72  * cleared, as entries will be in an inconsistent state.
73  *
74  * To reduce the level of work for a list that starts empty, we keep around a
75  * hint as to whether it was empty when we began, so we can avoid searching
76  * the list for entries to update.  Updates are O(n^2) due to searching for
77  * each entry before adding it.
78  */
79 int
80 memstat_sysctl_malloc(struct memory_type_list *list, int flags)
81 {
82 	struct malloc_type_stream_header *mtshp;
83 	struct malloc_type_header *mthp;
84 	struct malloc_type_stats *mtsp;
85 	struct memory_type *mtp;
86 	int count, hint_dontsearch, i, j, maxcpus;
87 	char *buffer, *p;
88 	size_t size;
89 
90 	hint_dontsearch = LIST_EMPTY(&list->mtl_list);
91 
92 	/*
93 	 * Query the number of CPUs, number of malloc types so that we can
94 	 * guess an initial buffer size.  We loop until we succeed or really
95 	 * fail.  Note that the value of maxcpus we query using sysctl is not
96 	 * the version we use when processing the real data -- that is read
97 	 * from the header.
98 	 */
99 retry:
100 	size = sizeof(maxcpus);
101 	if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) {
102 		if (errno == EACCES || errno == EPERM)
103 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
104 		else
105 			list->mtl_error = MEMSTAT_ERROR_DATAERROR;
106 		return (-1);
107 	}
108 	if (size != sizeof(maxcpus)) {
109 		list->mtl_error = MEMSTAT_ERROR_DATAERROR;
110 		return (-1);
111 	}
112 
113 	size = sizeof(count);
114 	if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) {
115 		if (errno == EACCES || errno == EPERM)
116 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
117 		else
118 			list->mtl_error = MEMSTAT_ERROR_VERSION;
119 		return (-1);
120 	}
121 	if (size != sizeof(count)) {
122 		list->mtl_error = MEMSTAT_ERROR_DATAERROR;
123 		return (-1);
124 	}
125 
126 	if (memstat_malloc_zone_init() == -1) {
127 		list->mtl_error = MEMSTAT_ERROR_VERSION;
128 		return (-1);
129 	}
130 
131 	size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) *
132 	    maxcpus);
133 
134 	buffer = malloc(size);
135 	if (buffer == NULL) {
136 		list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
137 		return (-1);
138 	}
139 
140 	if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) {
141 		/*
142 		 * XXXRW: ENOMEM is an ambiguous return, we should bound the
143 		 * number of loops, perhaps.
144 		 */
145 		if (errno == ENOMEM) {
146 			free(buffer);
147 			goto retry;
148 		}
149 		if (errno == EACCES || errno == EPERM)
150 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
151 		else
152 			list->mtl_error = MEMSTAT_ERROR_VERSION;
153 		free(buffer);
154 		return (-1);
155 	}
156 
157 	if (size == 0) {
158 		free(buffer);
159 		return (0);
160 	}
161 
162 	if (size < sizeof(*mtshp)) {
163 		list->mtl_error = MEMSTAT_ERROR_VERSION;
164 		free(buffer);
165 		return (-1);
166 	}
167 	p = buffer;
168 	mtshp = (struct malloc_type_stream_header *)p;
169 	p += sizeof(*mtshp);
170 
171 	if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) {
172 		list->mtl_error = MEMSTAT_ERROR_VERSION;
173 		free(buffer);
174 		return (-1);
175 	}
176 
177 	/*
178 	 * For the remainder of this function, we are quite trusting about
179 	 * the layout of structures and sizes, since we've determined we have
180 	 * a matching version and acceptable CPU count.
181 	 */
182 	maxcpus = mtshp->mtsh_maxcpus;
183 	count = mtshp->mtsh_count;
184 	for (i = 0; i < count; i++) {
185 		mthp = (struct malloc_type_header *)p;
186 		p += sizeof(*mthp);
187 
188 		if (hint_dontsearch == 0) {
189 			mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC,
190 			    mthp->mth_name);
191 		} else
192 			mtp = NULL;
193 		if (mtp == NULL)
194 			mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
195 			    mthp->mth_name, maxcpus);
196 		if (mtp == NULL) {
197 			_memstat_mtl_empty(list);
198 			free(buffer);
199 			list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
200 			return (-1);
201 		}
202 
203 		/*
204 		 * Reset the statistics on a current node.
205 		 */
206 		_memstat_mt_reset_stats(mtp, maxcpus);
207 
208 		for (j = 0; j < maxcpus; j++) {
209 			mtsp = (struct malloc_type_stats *)p;
210 			p += sizeof(*mtsp);
211 
212 			/*
213 			 * Sumarize raw statistics across CPUs into coalesced
214 			 * statistics.
215 			 */
216 			mtp->mt_memalloced += mtsp->mts_memalloced;
217 			mtp->mt_memfreed += mtsp->mts_memfreed;
218 			mtp->mt_numallocs += mtsp->mts_numallocs;
219 			mtp->mt_numfrees += mtsp->mts_numfrees;
220 			mtp->mt_sizemask |= mtsp->mts_size;
221 
222 			/*
223 			 * Copies of per-CPU statistics.
224 			 */
225 			mtp->mt_percpu_alloc[j].mtp_memalloced =
226 			    mtsp->mts_memalloced;
227 			mtp->mt_percpu_alloc[j].mtp_memfreed =
228 			    mtsp->mts_memfreed;
229 			mtp->mt_percpu_alloc[j].mtp_numallocs =
230 			    mtsp->mts_numallocs;
231 			mtp->mt_percpu_alloc[j].mtp_numfrees =
232 			    mtsp->mts_numfrees;
233 			mtp->mt_percpu_alloc[j].mtp_sizemask =
234 			    mtsp->mts_size;
235 		}
236 
237 		/*
238 		 * Derived cross-CPU statistics.
239 		 */
240 		mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
241 		mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
242 	}
243 
244 	free(buffer);
245 
246 	return (0);
247 }
248 
249 static int
250 kread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size,
251     size_t offset)
252 {
253 	ssize_t ret;
254 
255 	ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address,
256 	    size);
257 	if (ret < 0)
258 		return (MEMSTAT_ERROR_KVM);
259 	if ((size_t)ret != size)
260 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
261 	return (0);
262 }
263 
264 static int
265 kread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen)
266 {
267 	ssize_t ret;
268 	int i;
269 
270 	for (i = 0; i < buflen; i++) {
271 		ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) +
272 		    i, &(buffer[i]), sizeof(char));
273 		if (ret < 0)
274 			return (MEMSTAT_ERROR_KVM);
275 		if ((size_t)ret != sizeof(char))
276 			return (MEMSTAT_ERROR_KVM_SHORTREAD);
277 		if (buffer[i] == '\0')
278 			return (0);
279 	}
280 	/* Truncate. */
281 	buffer[i-1] = '\0';
282 	return (0);
283 }
284 
285 static int
286 kread_symbol(kvm_t *kvm, int index, void *address, size_t size,
287     size_t offset)
288 {
289 	ssize_t ret;
290 
291 	ret = kvm_read(kvm, namelist[index].n_value + offset, address, size);
292 	if (ret < 0)
293 		return (MEMSTAT_ERROR_KVM);
294 	if ((size_t)ret != size)
295 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
296 	return (0);
297 }
298 
299 static int
300 kread_zpcpu(kvm_t *kvm, u_long base, void *buf, size_t size, int cpu)
301 {
302 	ssize_t ret;
303 
304 	ret = kvm_read_zpcpu(kvm, base, buf, size, cpu);
305 	if (ret < 0)
306 		return (MEMSTAT_ERROR_KVM);
307 	if ((size_t)ret != size)
308 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
309 	return (0);
310 }
311 
312 int
313 memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
314 {
315 	struct memory_type *mtp;
316 	void *kmemstatistics;
317 	int hint_dontsearch, j, mp_maxcpus, mp_ncpus, ret;
318 	char name[MEMTYPE_MAXNAME];
319 	struct malloc_type_stats mts;
320 	struct malloc_type_internal *mtip;
321 	struct malloc_type type, *typep;
322 	kvm_t *kvm;
323 
324 	kvm = (kvm_t *)kvm_handle;
325 
326 	hint_dontsearch = LIST_EMPTY(&list->mtl_list);
327 
328 	if (kvm_nlist(kvm, namelist) != 0) {
329 		list->mtl_error = MEMSTAT_ERROR_KVM;
330 		return (-1);
331 	}
332 
333 	if (namelist[X_KMEMSTATISTICS].n_type == 0 ||
334 	    namelist[X_KMEMSTATISTICS].n_value == 0) {
335 		list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL;
336 		return (-1);
337 	}
338 
339 	ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus,
340 	    sizeof(mp_maxcpus), 0);
341 	if (ret != 0) {
342 		list->mtl_error = ret;
343 		return (-1);
344 	}
345 
346 	ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics,
347 	    sizeof(kmemstatistics), 0);
348 	if (ret != 0) {
349 		list->mtl_error = ret;
350 		return (-1);
351 	}
352 
353 	ret = memstat_malloc_zone_init_kvm(kvm);
354 	if (ret != 0) {
355 		list->mtl_error = ret;
356 		return (-1);
357 	}
358 
359 	mp_ncpus = kvm_getncpus(kvm);
360 
361 	for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) {
362 		ret = kread(kvm, typep, &type, sizeof(type), 0);
363 		if (ret != 0) {
364 			_memstat_mtl_empty(list);
365 			list->mtl_error = ret;
366 			return (-1);
367 		}
368 		ret = kread_string(kvm, (void *)type.ks_shortdesc, name,
369 		    MEMTYPE_MAXNAME);
370 		if (ret != 0) {
371 			_memstat_mtl_empty(list);
372 			list->mtl_error = ret;
373 			return (-1);
374 		}
375 		if (type.ks_version != M_VERSION) {
376 			warnx("type %s with unsupported version %lu; skipped",
377 			    name, type.ks_version);
378 			continue;
379 		}
380 
381 		/*
382 		 * Since our compile-time value for MAXCPU may differ from the
383 		 * kernel's, we populate our own array.
384 		 */
385 		mtip = &type.ks_mti;
386 
387 		if (hint_dontsearch == 0) {
388 			mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name);
389 		} else
390 			mtp = NULL;
391 		if (mtp == NULL)
392 			mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
393 			    name, mp_maxcpus);
394 		if (mtp == NULL) {
395 			_memstat_mtl_empty(list);
396 			list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
397 			return (-1);
398 		}
399 
400 		/*
401 		 * This logic is replicated from kern_malloc.c, and should
402 		 * be kept in sync.
403 		 */
404 		_memstat_mt_reset_stats(mtp, mp_maxcpus);
405 		for (j = 0; j < mp_ncpus; j++) {
406 			ret = kread_zpcpu(kvm, (u_long)mtip->mti_stats, &mts,
407 			    sizeof(mts), j);
408 			if (ret != 0) {
409 				_memstat_mtl_empty(list);
410 				list->mtl_error = ret;
411 				return (-1);
412 			}
413 			mtp->mt_memalloced += mts.mts_memalloced;
414 			mtp->mt_memfreed += mts.mts_memfreed;
415 			mtp->mt_numallocs += mts.mts_numallocs;
416 			mtp->mt_numfrees += mts.mts_numfrees;
417 			mtp->mt_sizemask |= mts.mts_size;
418 
419 			mtp->mt_percpu_alloc[j].mtp_memalloced =
420 			    mts.mts_memalloced;
421 			mtp->mt_percpu_alloc[j].mtp_memfreed =
422 			    mts.mts_memfreed;
423 			mtp->mt_percpu_alloc[j].mtp_numallocs =
424 			    mts.mts_numallocs;
425 			mtp->mt_percpu_alloc[j].mtp_numfrees =
426 			    mts.mts_numfrees;
427 			mtp->mt_percpu_alloc[j].mtp_sizemask =
428 			    mts.mts_size;
429 		}
430 		for (; j < mp_maxcpus; j++) {
431 			bzero(&mtp->mt_percpu_alloc[j],
432 			    sizeof(mtp->mt_percpu_alloc[0]));
433 		}
434 
435 		mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
436 		mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
437 	}
438 
439 	return (0);
440 }
441 
442 static int
443 memstat_malloc_zone_init(void)
444 {
445 	size_t size;
446 
447 	size = sizeof(memstat_malloc_zone_count);
448 	if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count,
449 	    &size, NULL, 0) < 0) {
450 		return (-1);
451 	}
452 
453 	if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) {
454 		return (-1);
455 	}
456 
457 	size = sizeof(memstat_malloc_zone_sizes);
458 	if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes,
459 	    &size, NULL, 0) < 0) {
460 		return (-1);
461 	}
462 
463 	return (0);
464 }
465 
466 /*
467  * Copied from kern_malloc.c
468  *
469  * kz_zone is an array sized at compilation time, the size is exported in
470  * "numzones". Below we need to iterate kz_size.
471  */
472 struct memstat_kmemzone {
473 	int kz_size;
474 	const char *kz_name;
475 	void *kz_zone[1];
476 };
477 
478 static int
479 memstat_malloc_zone_init_kvm(kvm_t *kvm)
480 {
481 	struct memstat_kmemzone *kmemzones, *kz;
482 	int numzones, objsize, allocsize, ret;
483 	int i;
484 
485 	ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT,
486 	    &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0);
487 	if (ret != 0) {
488 		return (ret);
489 	}
490 
491 	ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0);
492 	if (ret != 0) {
493 		return (ret);
494 	}
495 
496 	objsize = __offsetof(struct memstat_kmemzone, kz_zone) +
497 	    sizeof(void *) * numzones;
498 
499 	allocsize = objsize * memstat_malloc_zone_count;
500 	kmemzones = malloc(allocsize);
501 	if (kmemzones == NULL) {
502 		return (MEMSTAT_ERROR_NOMEMORY);
503 	}
504 	ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0);
505 	if (ret != 0) {
506 		free(kmemzones);
507 		return (ret);
508 	}
509 
510 	kz = kmemzones;
511 	for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) {
512 		memstat_malloc_zone_sizes[i] = kz->kz_size;
513 		kz = (struct memstat_kmemzone *)((char *)kz + objsize);
514 	}
515 
516 	free(kmemzones);
517 	return (0);
518 }
519 
520 size_t
521 memstat_malloc_zone_get_count(void)
522 {
523 
524 	return (memstat_malloc_zone_count);
525 }
526 
527 size_t
528 memstat_malloc_zone_get_size(size_t n)
529 {
530 
531 	if (n >= nitems(memstat_malloc_zone_sizes)) {
532 		return (-1);
533 	}
534 
535 	return (memstat_malloc_zone_sizes[n]);
536 }
537 
538 int
539 memstat_malloc_zone_used(const struct memory_type *mtp, size_t n)
540 {
541 
542 	if (memstat_get_sizemask(mtp) & (1 << n))
543 		return (1);
544 
545 	return (0);
546 }
547