xref: /freebsd/contrib/jemalloc/src/malloc_io.c (revision d6b92ffa) 
1 #define JEMALLOC_MALLOC_IO_C_
2 #include "jemalloc/internal/jemalloc_preamble.h"
3 #include "jemalloc/internal/jemalloc_internal_includes.h"
4 
5 #include "jemalloc/internal/malloc_io.h"
6 #include "jemalloc/internal/util.h"
7 
8 #ifdef assert
9 #  undef assert
10 #endif
11 #ifdef not_reached
12 #  undef not_reached
13 #endif
14 #ifdef not_implemented
15 #  undef not_implemented
16 #endif
17 #ifdef assert_not_implemented
18 #  undef assert_not_implemented
19 #endif
20 
21 /*
22  * Define simple versions of assertion macros that won't recurse in case
23  * of assertion failures in malloc_*printf().
24  */
25 #define assert(e) do {							\
26 	if (config_debug && !(e)) {					\
27 		malloc_write("<jemalloc>: Failed assertion\n");		\
28 		abort();						\
29 	}								\
30 } while (0)
31 
32 #define not_reached() do {						\
33 	if (config_debug) {						\
34 		malloc_write("<jemalloc>: Unreachable code reached\n");	\
35 		abort();						\
36 	}								\
37 	unreachable();							\
38 } while (0)
39 
40 #define not_implemented() do {						\
41 	if (config_debug) {						\
42 		malloc_write("<jemalloc>: Not implemented\n");		\
43 		abort();						\
44 	}								\
45 } while (0)
46 
47 #define assert_not_implemented(e) do {					\
48 	if (unlikely(config_debug && !(e))) {				\
49 		not_implemented();					\
50 	}								\
51 } while (0)
52 
53 /******************************************************************************/
54 /* Function prototypes for non-inline static functions. */
55 
56 static void wrtmessage(void *cbopaque, const char *s);
57 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
58 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
59     size_t *slen_p);
60 #define D2S_BUFSIZE (1 + U2S_BUFSIZE)
61 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
62 #define O2S_BUFSIZE (1 + U2S_BUFSIZE)
63 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
64 #define X2S_BUFSIZE (2 + U2S_BUFSIZE)
65 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
66     size_t *slen_p);
67 
68 /******************************************************************************/
69 
70 /* malloc_message() setup. */
71 static void
72 wrtmessage(void *cbopaque, const char *s) {
73 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
74 	/*
75 	 * Use syscall(2) rather than write(2) when possible in order to avoid
76 	 * the possibility of memory allocation within libc.  This is necessary
77 	 * on FreeBSD; most operating systems do not have this problem though.
78 	 *
79 	 * syscall() returns long or int, depending on platform, so capture the
80 	 * unused result in the widest plausible type to avoid compiler
81 	 * warnings.
82 	 */
83 	UNUSED long result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
84 #else
85 	UNUSED ssize_t result = write(STDERR_FILENO, s, strlen(s));
86 #endif
87 }
88 
89 JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);
90 
91 JEMALLOC_ATTR(visibility("hidden"))
92 void
93 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
94 
95 	wrtmessage(NULL, s1);
96 	wrtmessage(NULL, s2);
97 	wrtmessage(NULL, s3);
98 	wrtmessage(NULL, s4);
99 }
100 
101 void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
102     const char *s4) = wrtmessage_1_0;
103 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
104 
105 /*
106  * Wrapper around malloc_message() that avoids the need for
107  * je_malloc_message(...) throughout the code.
108  */
109 void
110 malloc_write(const char *s) {
111 	if (je_malloc_message != NULL) {
112 		je_malloc_message(NULL, s);
113 	} else {
114 		wrtmessage(NULL, s);
115 	}
116 }
117 
118 /*
119  * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
120  * provide a wrapper.
121  */
122 int
123 buferror(int err, char *buf, size_t buflen) {
124 #ifdef _WIN32
125 	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
126 	    (LPSTR)buf, (DWORD)buflen, NULL);
127 	return 0;
128 #elif defined(__GLIBC__) && defined(_GNU_SOURCE)
129 	char *b = strerror_r(err, buf, buflen);
130 	if (b != buf) {
131 		strncpy(buf, b, buflen);
132 		buf[buflen-1] = '\0';
133 	}
134 	return 0;
135 #else
136 	return strerror_r(err, buf, buflen);
137 #endif
138 }
139 
140 uintmax_t
141 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
142 	uintmax_t ret, digit;
143 	unsigned b;
144 	bool neg;
145 	const char *p, *ns;
146 
147 	p = nptr;
148 	if (base < 0 || base == 1 || base > 36) {
149 		ns = p;
150 		set_errno(EINVAL);
151 		ret = UINTMAX_MAX;
152 		goto label_return;
153 	}
154 	b = base;
155 
156 	/* Swallow leading whitespace and get sign, if any. */
157 	neg = false;
158 	while (true) {
159 		switch (*p) {
160 		case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
161 			p++;
162 			break;
163 		case '-':
164 			neg = true;
165 			/* Fall through. */
166 		case '+':
167 			p++;
168 			/* Fall through. */
169 		default:
170 			goto label_prefix;
171 		}
172 	}
173 
174 	/* Get prefix, if any. */
175 	label_prefix:
176 	/*
177 	 * Note where the first non-whitespace/sign character is so that it is
178 	 * possible to tell whether any digits are consumed (e.g., "  0" vs.
179 	 * "  -x").
180 	 */
181 	ns = p;
182 	if (*p == '0') {
183 		switch (p[1]) {
184 		case '0': case '1': case '2': case '3': case '4': case '5':
185 		case '6': case '7':
186 			if (b == 0) {
187 				b = 8;
188 			}
189 			if (b == 8) {
190 				p++;
191 			}
192 			break;
193 		case 'X': case 'x':
194 			switch (p[2]) {
195 			case '0': case '1': case '2': case '3': case '4':
196 			case '5': case '6': case '7': case '8': case '9':
197 			case 'A': case 'B': case 'C': case 'D': case 'E':
198 			case 'F':
199 			case 'a': case 'b': case 'c': case 'd': case 'e':
200 			case 'f':
201 				if (b == 0) {
202 					b = 16;
203 				}
204 				if (b == 16) {
205 					p += 2;
206 				}
207 				break;
208 			default:
209 				break;
210 			}
211 			break;
212 		default:
213 			p++;
214 			ret = 0;
215 			goto label_return;
216 		}
217 	}
218 	if (b == 0) {
219 		b = 10;
220 	}
221 
222 	/* Convert. */
223 	ret = 0;
224 	while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
225 	    || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
226 	    || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
227 		uintmax_t pret = ret;
228 		ret *= b;
229 		ret += digit;
230 		if (ret < pret) {
231 			/* Overflow. */
232 			set_errno(ERANGE);
233 			ret = UINTMAX_MAX;
234 			goto label_return;
235 		}
236 		p++;
237 	}
238 	if (neg) {
239 		ret = (uintmax_t)(-((intmax_t)ret));
240 	}
241 
242 	if (p == ns) {
243 		/* No conversion performed. */
244 		set_errno(EINVAL);
245 		ret = UINTMAX_MAX;
246 		goto label_return;
247 	}
248 
249 label_return:
250 	if (endptr != NULL) {
251 		if (p == ns) {
252 			/* No characters were converted. */
253 			*endptr = (char *)nptr;
254 		} else {
255 			*endptr = (char *)p;
256 		}
257 	}
258 	return ret;
259 }
260 
261 static char *
262 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
263 	unsigned i;
264 
265 	i = U2S_BUFSIZE - 1;
266 	s[i] = '\0';
267 	switch (base) {
268 	case 10:
269 		do {
270 			i--;
271 			s[i] = "0123456789"[x % (uint64_t)10];
272 			x /= (uint64_t)10;
273 		} while (x > 0);
274 		break;
275 	case 16: {
276 		const char *digits = (uppercase)
277 		    ? "0123456789ABCDEF"
278 		    : "0123456789abcdef";
279 
280 		do {
281 			i--;
282 			s[i] = digits[x & 0xf];
283 			x >>= 4;
284 		} while (x > 0);
285 		break;
286 	} default: {
287 		const char *digits = (uppercase)
288 		    ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
289 		    : "0123456789abcdefghijklmnopqrstuvwxyz";
290 
291 		assert(base >= 2 && base <= 36);
292 		do {
293 			i--;
294 			s[i] = digits[x % (uint64_t)base];
295 			x /= (uint64_t)base;
296 		} while (x > 0);
297 	}}
298 
299 	*slen_p = U2S_BUFSIZE - 1 - i;
300 	return &s[i];
301 }
302 
303 static char *
304 d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
305 	bool neg;
306 
307 	if ((neg = (x < 0))) {
308 		x = -x;
309 	}
310 	s = u2s(x, 10, false, s, slen_p);
311 	if (neg) {
312 		sign = '-';
313 	}
314 	switch (sign) {
315 	case '-':
316 		if (!neg) {
317 			break;
318 		}
319 		/* Fall through. */
320 	case ' ':
321 	case '+':
322 		s--;
323 		(*slen_p)++;
324 		*s = sign;
325 		break;
326 	default: not_reached();
327 	}
328 	return s;
329 }
330 
331 static char *
332 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
333 	s = u2s(x, 8, false, s, slen_p);
334 	if (alt_form && *s != '0') {
335 		s--;
336 		(*slen_p)++;
337 		*s = '0';
338 	}
339 	return s;
340 }
341 
342 static char *
343 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
344 	s = u2s(x, 16, uppercase, s, slen_p);
345 	if (alt_form) {
346 		s -= 2;
347 		(*slen_p) += 2;
348 		memcpy(s, uppercase ? "0X" : "0x", 2);
349 	}
350 	return s;
351 }
352 
353 size_t
354 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
355 	size_t i;
356 	const char *f;
357 
358 #define APPEND_C(c) do {						\
359 	if (i < size) {							\
360 		str[i] = (c);						\
361 	}								\
362 	i++;								\
363 } while (0)
364 #define APPEND_S(s, slen) do {						\
365 	if (i < size) {							\
366 		size_t cpylen = (slen <= size - i) ? slen : size - i;	\
367 		memcpy(&str[i], s, cpylen);				\
368 	}								\
369 	i += slen;							\
370 } while (0)
371 #define APPEND_PADDED_S(s, slen, width, left_justify) do {		\
372 	/* Left padding. */						\
373 	size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?	\
374 	    (size_t)width - slen : 0);					\
375 	if (!left_justify && pad_len != 0) {				\
376 		size_t j;						\
377 		for (j = 0; j < pad_len; j++) {				\
378 			APPEND_C(' ');					\
379 		}							\
380 	}								\
381 	/* Value. */							\
382 	APPEND_S(s, slen);						\
383 	/* Right padding. */						\
384 	if (left_justify && pad_len != 0) {				\
385 		size_t j;						\
386 		for (j = 0; j < pad_len; j++) {				\
387 			APPEND_C(' ');					\
388 		}							\
389 	}								\
390 } while (0)
391 #define GET_ARG_NUMERIC(val, len) do {					\
392 	switch (len) {							\
393 	case '?':							\
394 		val = va_arg(ap, int);					\
395 		break;							\
396 	case '?' | 0x80:						\
397 		val = va_arg(ap, unsigned int);				\
398 		break;							\
399 	case 'l':							\
400 		val = va_arg(ap, long);					\
401 		break;							\
402 	case 'l' | 0x80:						\
403 		val = va_arg(ap, unsigned long);			\
404 		break;							\
405 	case 'q':							\
406 		val = va_arg(ap, long long);				\
407 		break;							\
408 	case 'q' | 0x80:						\
409 		val = va_arg(ap, unsigned long long);			\
410 		break;							\
411 	case 'j':							\
412 		val = va_arg(ap, intmax_t);				\
413 		break;							\
414 	case 'j' | 0x80:						\
415 		val = va_arg(ap, uintmax_t);				\
416 		break;							\
417 	case 't':							\
418 		val = va_arg(ap, ptrdiff_t);				\
419 		break;							\
420 	case 'z':							\
421 		val = va_arg(ap, ssize_t);				\
422 		break;							\
423 	case 'z' | 0x80:						\
424 		val = va_arg(ap, size_t);				\
425 		break;							\
426 	case 'p': /* Synthetic; used for %p. */				\
427 		val = va_arg(ap, uintptr_t);				\
428 		break;							\
429 	default:							\
430 		not_reached();						\
431 		val = 0;						\
432 	}								\
433 } while (0)
434 
435 	i = 0;
436 	f = format;
437 	while (true) {
438 		switch (*f) {
439 		case '\0': goto label_out;
440 		case '%': {
441 			bool alt_form = false;
442 			bool left_justify = false;
443 			bool plus_space = false;
444 			bool plus_plus = false;
445 			int prec = -1;
446 			int width = -1;
447 			unsigned char len = '?';
448 			char *s;
449 			size_t slen;
450 
451 			f++;
452 			/* Flags. */
453 			while (true) {
454 				switch (*f) {
455 				case '#':
456 					assert(!alt_form);
457 					alt_form = true;
458 					break;
459 				case '-':
460 					assert(!left_justify);
461 					left_justify = true;
462 					break;
463 				case ' ':
464 					assert(!plus_space);
465 					plus_space = true;
466 					break;
467 				case '+':
468 					assert(!plus_plus);
469 					plus_plus = true;
470 					break;
471 				default: goto label_width;
472 				}
473 				f++;
474 			}
475 			/* Width. */
476 			label_width:
477 			switch (*f) {
478 			case '*':
479 				width = va_arg(ap, int);
480 				f++;
481 				if (width < 0) {
482 					left_justify = true;
483 					width = -width;
484 				}
485 				break;
486 			case '0': case '1': case '2': case '3': case '4':
487 			case '5': case '6': case '7': case '8': case '9': {
488 				uintmax_t uwidth;
489 				set_errno(0);
490 				uwidth = malloc_strtoumax(f, (char **)&f, 10);
491 				assert(uwidth != UINTMAX_MAX || get_errno() !=
492 				    ERANGE);
493 				width = (int)uwidth;
494 				break;
495 			} default:
496 				break;
497 			}
498 			/* Width/precision separator. */
499 			if (*f == '.') {
500 				f++;
501 			} else {
502 				goto label_length;
503 			}
504 			/* Precision. */
505 			switch (*f) {
506 			case '*':
507 				prec = va_arg(ap, int);
508 				f++;
509 				break;
510 			case '0': case '1': case '2': case '3': case '4':
511 			case '5': case '6': case '7': case '8': case '9': {
512 				uintmax_t uprec;
513 				set_errno(0);
514 				uprec = malloc_strtoumax(f, (char **)&f, 10);
515 				assert(uprec != UINTMAX_MAX || get_errno() !=
516 				    ERANGE);
517 				prec = (int)uprec;
518 				break;
519 			}
520 			default: break;
521 			}
522 			/* Length. */
523 			label_length:
524 			switch (*f) {
525 			case 'l':
526 				f++;
527 				if (*f == 'l') {
528 					len = 'q';
529 					f++;
530 				} else {
531 					len = 'l';
532 				}
533 				break;
534 			case 'q': case 'j': case 't': case 'z':
535 				len = *f;
536 				f++;
537 				break;
538 			default: break;
539 			}
540 			/* Conversion specifier. */
541 			switch (*f) {
542 			case '%':
543 				/* %% */
544 				APPEND_C(*f);
545 				f++;
546 				break;
547 			case 'd': case 'i': {
548 				intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
549 				char buf[D2S_BUFSIZE];
550 
551 				GET_ARG_NUMERIC(val, len);
552 				s = d2s(val, (plus_plus ? '+' : (plus_space ?
553 				    ' ' : '-')), buf, &slen);
554 				APPEND_PADDED_S(s, slen, width, left_justify);
555 				f++;
556 				break;
557 			} case 'o': {
558 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
559 				char buf[O2S_BUFSIZE];
560 
561 				GET_ARG_NUMERIC(val, len | 0x80);
562 				s = o2s(val, alt_form, buf, &slen);
563 				APPEND_PADDED_S(s, slen, width, left_justify);
564 				f++;
565 				break;
566 			} case 'u': {
567 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
568 				char buf[U2S_BUFSIZE];
569 
570 				GET_ARG_NUMERIC(val, len | 0x80);
571 				s = u2s(val, 10, false, buf, &slen);
572 				APPEND_PADDED_S(s, slen, width, left_justify);
573 				f++;
574 				break;
575 			} case 'x': case 'X': {
576 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
577 				char buf[X2S_BUFSIZE];
578 
579 				GET_ARG_NUMERIC(val, len | 0x80);
580 				s = x2s(val, alt_form, *f == 'X', buf, &slen);
581 				APPEND_PADDED_S(s, slen, width, left_justify);
582 				f++;
583 				break;
584 			} case 'c': {
585 				unsigned char val;
586 				char buf[2];
587 
588 				assert(len == '?' || len == 'l');
589 				assert_not_implemented(len != 'l');
590 				val = va_arg(ap, int);
591 				buf[0] = val;
592 				buf[1] = '\0';
593 				APPEND_PADDED_S(buf, 1, width, left_justify);
594 				f++;
595 				break;
596 			} case 's':
597 				assert(len == '?' || len == 'l');
598 				assert_not_implemented(len != 'l');
599 				s = va_arg(ap, char *);
600 				slen = (prec < 0) ? strlen(s) : (size_t)prec;
601 				APPEND_PADDED_S(s, slen, width, left_justify);
602 				f++;
603 				break;
604 			case 'p': {
605 				uintmax_t val;
606 				char buf[X2S_BUFSIZE];
607 
608 				GET_ARG_NUMERIC(val, 'p');
609 				s = x2s(val, true, false, buf, &slen);
610 				APPEND_PADDED_S(s, slen, width, left_justify);
611 				f++;
612 				break;
613 			} default: not_reached();
614 			}
615 			break;
616 		} default: {
617 			APPEND_C(*f);
618 			f++;
619 			break;
620 		}}
621 	}
622 	label_out:
623 	if (i < size) {
624 		str[i] = '\0';
625 	} else {
626 		str[size - 1] = '\0';
627 	}
628 
629 #undef APPEND_C
630 #undef APPEND_S
631 #undef APPEND_PADDED_S
632 #undef GET_ARG_NUMERIC
633 	return i;
634 }
635 
636 JEMALLOC_FORMAT_PRINTF(3, 4)
637 size_t
638 malloc_snprintf(char *str, size_t size, const char *format, ...) {
639 	size_t ret;
640 	va_list ap;
641 
642 	va_start(ap, format);
643 	ret = malloc_vsnprintf(str, size, format, ap);
644 	va_end(ap);
645 
646 	return ret;
647 }
648 
649 void
650 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
651     const char *format, va_list ap) {
652 	char buf[MALLOC_PRINTF_BUFSIZE];
653 
654 	if (write_cb == NULL) {
655 		/*
656 		 * The caller did not provide an alternate write_cb callback
657 		 * function, so use the default one.  malloc_write() is an
658 		 * inline function, so use malloc_message() directly here.
659 		 */
660 		write_cb = (je_malloc_message != NULL) ? je_malloc_message :
661 		    wrtmessage;
662 		cbopaque = NULL;
663 	}
664 
665 	malloc_vsnprintf(buf, sizeof(buf), format, ap);
666 	write_cb(cbopaque, buf);
667 }
668 
669 /*
670  * Print to a callback function in such a way as to (hopefully) avoid memory
671  * allocation.
672  */
673 JEMALLOC_FORMAT_PRINTF(3, 4)
674 void
675 malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
676     const char *format, ...) {
677 	va_list ap;
678 
679 	va_start(ap, format);
680 	malloc_vcprintf(write_cb, cbopaque, format, ap);
681 	va_end(ap);
682 }
683 
684 /* Print to stderr in such a way as to avoid memory allocation. */
685 JEMALLOC_FORMAT_PRINTF(1, 2)
686 void
687 malloc_printf(const char *format, ...) {
688 	va_list ap;
689 
690 	va_start(ap, format);
691 	malloc_vcprintf(NULL, NULL, format, ap);
692 	va_end(ap);
693 }
694 
695 /*
696  * Restore normal assertion macros, in order to make it possible to compile all
697  * C files as a single concatenation.
698  */
699 #undef assert
700 #undef not_reached
701 #undef not_implemented
702 #undef assert_not_implemented
703 #include "jemalloc/internal/assert.h"
704