1 /*
2  * libiio - Library for interfacing industrial I/O (IIO) devices
3  *
4  * Copyright (C) 2014 Analog Devices, Inc.
5  * Author: Paul Cercueil <paul.cercueil@analog.com>
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * */
18 
19 #define _DEFAULT_SOURCE
20 
21 #include <getopt.h>
22 #include <iio.h>
23 #include <signal.h>
24 #include <stdio.h>
25 #include <string.h>
26 #include <pthread.h>
27 #include <unistd.h>
28 #include <errno.h>
29 #include <limits.h>
30 #include <sys/time.h>
31 
32 #ifdef __APPLE__
33 	/* Needed for sysctlbyname */
34 #include <sys/sysctl.h>
35 #endif
36 
37 #include "iio_common.h"
38 
39 #define MY_NAME "iio_stresstest"
40 
41 #define SAMPLES_PER_READ 256
42 #define NUM_TIMESTAMPS (16*1024)
43 
getNumCores(void)44 static int getNumCores(void) {
45 #ifdef _WIN32
46 	SYSTEM_INFO sysinfo;
47 	GetSystemInfo(&sysinfo);
48 	return sysinfo.dwNumberOfProcessors;
49 #elif __APPLE__
50 	int count;
51 	size_t count_len = sizeof(count);
52 	sysctlbyname("hw.logicalcpu", &count, &count_len, NULL, 0);
53 	return count;
54 #else
55 	return sysconf(_SC_NPROCESSORS_ONLN);
56 #endif
57 }
58 
59 
60 /* Code snippet insired by Nick Strupat
61  * https://stackoverflow.com/questions/794632/programmatically-get-the-cache-line-size
62  * released under the "Feel free to do whatever you want with it." license.
63  */
cache_line_size(void)64 static size_t cache_line_size(void)
65 {
66 	size_t cacheline = 0;
67 
68 #ifdef _WIN32
69 	DWORD buffer_size = 0;
70 	DWORD i = 0;
71 	SYSTEM_LOGICAL_PROCESSOR_INFORMATION * buffer = 0;
72 
73 	GetLogicalProcessorInformation(0, &buffer_size);
74 	buffer = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION *)malloc(buffer_size);
75 	GetLogicalProcessorInformation(&buffer[0], &buffer_size);
76 
77 	for (i = 0; i != buffer_size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); ++i) {
78 		if (buffer[i].Relationship == RelationCache && buffer[i].Cache.Level == 1) {
79 			cacheline = buffer[i].Cache.LineSize;
80 			break;
81 		}
82 	}
83 	free(buffer);
84 
85 #elif __APPLE__
86 	size_t sizeof_line_size = sizeof(cacheline);
87 	sysctlbyname("hw.cachelinesize", &cacheline, &sizeof_line_size, 0, 0);
88 
89 #elif __linux__
90 	FILE * p = 0;
91 	int ret;
92 
93 	p = fopen("/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size", "r");
94 	if (p) {
95 		ret = fscanf(p, "%zu", &cacheline);
96 		fclose(p);
97 		if (ret != 1)
98 			cacheline = 0;
99 	}
100 #endif
101 	return cacheline;
102 }
103 
104 static const struct option options[] = {
105 	{"help", no_argument, 0, 'h'},
106 	{"uri", required_argument, 0, 'u'},
107 	{"buffer-size", required_argument, 0, 'b'},
108 	{"samples", required_argument, 0, 's' },
109 	{"timeout", required_argument, 0, 't'},
110 	{"Threads", required_argument, 0, 'T'},
111 	{"verbose", no_argument, 0, 'v'},
112 	{0, 0, 0, 0},
113 };
114 
115 static const char *options_descriptions[] = {
116 	"[-n <hostname>] [-u <vid>:<pid>] [-t <trigger>] [-b <buffer-size>] [-s <samples>]"
117 		"<iio_device> [<channel> ...]",
118 	"Show this help and quit.",
119 	"Use the context at the provided URI.",
120 	"Size of the capture buffer. Default is 256.",
121 	"Number of samples to capture, 0 = infinite. Default is 0.",
122 	"Time to wait (in s) between stopping all threads",
123 	"Number of Threads",
124 	"Increase verbosity (-vv and -vvv for more)",
125 };
126 
127 static bool app_running = true;
128 static bool threads_running = true;
129 static int exit_code = EXIT_SUCCESS;
130 
compare_timeval(const void * a,const void * b)131 static int compare_timeval(const void *a, const void *b)
132 {
133 	const struct timeval *t1 = (struct timeval *)a;
134 	const struct timeval *t2 = (struct timeval *)b;
135 
136 	if (t1->tv_sec < t2->tv_sec)
137 		return -1;
138 	if (t1->tv_sec > t2->tv_sec)
139 		return 1;
140 
141 	/* only way to get here is if *t1.tv_sec == *t2.tv_sec */
142 	if (t1->tv_usec < t2->tv_usec)
143 		return -1;
144 	if (t1->tv_usec > t2->tv_usec)
145 		return 1;
146 
147 	/* must be same */
148 	return 0;
149 }
quit_all(int sig)150 static void quit_all(int sig)
151 {
152 	exit_code = sig;
153 	app_running = false;
154 	if (sig == SIGSEGV) {
155 		fprintf(stderr, "fatal error SIGSEGV, break out gdb\n");
156 		abort();
157 	}
158 }
159 
set_handler(int signal_nb,void (* handler)(int))160 static void set_handler(int signal_nb, void (*handler)(int))
161 {
162 #ifdef _WIN32
163 	signal(signal_nb, handler);
164 #else
165 	struct sigaction sig;
166 	sigaction(signal_nb, NULL, &sig);
167 	sig.sa_handler = handler;
168 	sigaction(signal_nb, &sig, NULL);
169 #endif
170 }
171 
get_device(const struct iio_context * ctx,const char * id)172 static struct iio_device * get_device(const struct iio_context *ctx,
173 		const char *id)
174 {
175 	unsigned int i, nb_devices = iio_context_get_devices_count(ctx);
176 	struct iio_device *device;
177 
178 	for (i = 0; i < nb_devices; i++) {
179 		const char *name;
180 		device = iio_context_get_device(ctx, i);
181 		name = iio_device_get_name(device);
182 		if (name && !strcmp(name, id))
183 			break;
184 		if (!strcmp(id, iio_device_get_id(device)))
185 			break;
186 	}
187 
188 	if (i < nb_devices)
189 		return device;
190 
191 	fprintf(stderr, "Device %s not found\n", id);
192 	return NULL;
193 }
194 
195 enum verbosity {
196 	QUIET,
197 	SUMMARY,
198 	VERBOSE,
199 	VERYVERBOSE,
200 };
201 
202 struct info {
203 	int argc;
204 	char **argv;
205 	enum backend back;
206 	enum verbosity verbose;
207 
208 	int uri_index, device_index, arg_index;
209 	unsigned int buffer_size, timeout;
210 	unsigned int num_threads;
211 	pthread_t *tid;
212 	unsigned int *starts, *buffers, *refills;
213 	pthread_t *threads;
214 	struct timeval **start;
215 };
216 
thread_err(int id,ssize_t ret,char * what)217 static void thread_err(int id, ssize_t ret, char * what)
218 {
219 	if (ret < 0) {
220 		char err_str[1024];
221 		iio_strerror(-ret, err_str, sizeof(err_str)); \
222 		fprintf(stderr, "%i : IIO ERROR : %s : %s (%zd)\n", id, what, err_str, ret); \
223 	}
224 }
225 
client_thread(void * data)226 static void *client_thread(void *data)
227 {
228 	struct info *info = data;
229 	struct iio_context *ctx;
230 	struct iio_buffer *buffer;
231 	unsigned int i, nb_channels, duration;
232 	struct iio_device *dev;
233 	struct timeval start, end;
234 	int id = -1, stamp, r_errno;
235 	ssize_t ret;
236 
237 	/* Find my ID */
238 	for (i = 0; i < info->num_threads; i++) {
239 		if (pthread_equal(info->threads[i], pthread_self())) {
240 			info->tid[i] = pthread_self();
241 			id = i;
242 			break;
243 		}
244 	}
245 
246 	if (info->verbose == VERYVERBOSE)
247 		printf("%2d: Entered\n", id);
248 
249 	stamp = 0;
250 	while (stamp < NUM_TIMESTAMPS && info->start[id][stamp].tv_sec) {
251 		stamp++;
252 	}
253 
254 	while (app_running && threads_running) {
255 		gettimeofday(&start, NULL);
256 		do {
257 			errno = 0;
258 			if (info->uri_index) {
259 				ctx = iio_create_context_from_uri(info->argv[info->uri_index]);
260 			} else {
261 				ctx = iio_create_default_context();
262 			}
263 			r_errno = errno;
264 			gettimeofday(&end, NULL);
265 
266 			duration = ((end.tv_sec - start.tv_sec) * 1000) +
267 					((end.tv_usec - start.tv_usec) / 1000);
268 		} while (threads_running && !ctx && duration < info->timeout);
269 
270 		if (!ctx) {
271 			thread_err(id, r_errno, "Unable to create IIO context");
272 			goto thread_fail;
273 		}
274 
275 		/* store the timestamp of the context creation */
276 		info->start[id][stamp].tv_sec = end.tv_sec;
277 		info->start[id][stamp].tv_usec = end.tv_usec;
278 		stamp++;
279 		if (stamp > NUM_TIMESTAMPS - 10 )
280 			threads_running = false;
281 
282 		/* started another context */
283 		info->starts[id]++;
284 
285 		ret = iio_context_set_timeout(ctx, UINT_MAX);
286 		thread_err(id, ret, "iio_context_set_timeout failed");
287 
288 		dev = get_device(ctx, info->argv[info->arg_index + 1]);
289 		if (!dev) {
290 			iio_context_destroy(ctx);
291 			goto thread_fail;
292 		}
293 
294 		nb_channels = iio_device_get_channels_count(dev);
295 
296 		if (info->argc == info->arg_index + 2) {
297 			/* Enable all channels */
298 			for (i = 0; i < nb_channels; i++)
299 				iio_channel_enable(iio_device_get_channel(dev, i));
300 		} else {
301 			for (i = 0; i < nb_channels; i++) {
302 				unsigned int j;
303 				struct iio_channel *ch = iio_device_get_channel(dev, i);
304 				for (j = info->arg_index + 2; j < (unsigned int) info->argc; j++) {
305 					const char *n = iio_channel_get_name(ch);
306 					if (!strcmp(info->argv[j], iio_channel_get_id(ch)) ||
307 							(n && !strcmp(n, info->argv[j])))
308 						iio_channel_enable(ch);
309 				}
310 			}
311 		}
312 
313 		if (info->verbose == VERYVERBOSE)
314 			printf("%2d: Running\n", id);
315 
316 		i = 0;
317 		while (threads_running || i == 0) {
318 			info->buffers[id]++;
319 			buffer = iio_device_create_buffer(dev, info->buffer_size, false);
320 			if (!buffer) {
321 				struct timespec wait;
322 				wait.tv_sec = 0;
323 				wait.tv_nsec = (1 * 1000);
324 				thread_err(id, errno, "iio_device_create_buffer failed");
325 				nanosleep(&wait, &wait);
326 				continue;
327 			}
328 
329 			while (threads_running || i == 0) {
330 				ret = iio_buffer_refill(buffer);
331 				thread_err(id, ret, "iio_buffer_refill failed");
332 				if (ret < 0) {
333 					threads_running = 0;
334 					break;
335 				}
336 				info->refills[id]++;
337 				i = 1;
338 
339 				/* depending on backend, do more */
340 				if(info->back == IIO_USB && rand() % 3 == 0)
341 					break;
342 				else if (info->back == IIO_NETWORK && rand() % 5 == 0)
343 					break;
344 				else if (rand() % 10 == 0)
345 					break;
346 			}
347 			iio_buffer_destroy(buffer);
348 
349 			/* depending on backend, do more */
350 			if(info->back == IIO_USB) {
351 				break;
352 			} else if (info->back == IIO_NETWORK) {
353 				if (rand() % 5 == 0)
354 					break;
355 			} else {
356 				if (rand() % 10 == 0)
357 					break;
358 			}
359 		}
360 
361 		iio_context_destroy(ctx);
362 		if (info->verbose == VERYVERBOSE)
363 			printf("%2d: Stopping\n", id);
364 
365 		/* 1 in 10, (or with above loops, 1 in 1000 stop */
366 		if (rand() % 100 == 0) {
367 			break;
368 		}
369 	}
370 
371 	if (info->verbose == VERYVERBOSE)
372 		printf("%2d: Stopped normal\n", id);
373 	info->tid[id] = 0;
374 	info->start[id][stamp].tv_sec = 0; info->start[id][stamp].tv_usec = 0;
375 	return (void *)0;
376 
377 thread_fail:
378 	if (info->verbose == VERYVERBOSE)
379 		printf("%2d: Stopped via error\n", id);
380 	info->tid[id] = 0;
381 	info->start[id][stamp].tv_sec = 0; info->start[id][stamp].tv_usec = 0;
382 	return (void *)EXIT_FAILURE;
383 }
384 
main(int argc,char ** argv)385 int main(int argc, char **argv)
386 {
387 	sigset_t set, oldset;
388 	struct info info;
389 	int option_index;
390 	unsigned int i, duration;
391 	int c, pret;
392 	struct timeval start, end, s_loop;
393 	void **ret;
394 	size_t min_samples;
395 
396 #ifndef _WIN32
397 	set_handler(SIGHUP, &quit_all);
398 	set_handler(SIGPIPE, &quit_all);
399 #endif
400 	set_handler(SIGINT, &quit_all);
401 	set_handler(SIGSEGV, &quit_all);
402 	set_handler(SIGTERM, &quit_all);
403 
404 	info.num_threads = getNumCores() * 4;
405 	info.buffer_size = SAMPLES_PER_READ;
406 	info.arg_index = 0;
407 	info.uri_index = 0;
408 	info.timeout = UINT_MAX;
409 	info.verbose = QUIET;
410 	info.argc = argc;
411 	info.argv = argv;
412 
413 	min_samples = cache_line_size();
414 	if(!min_samples)
415 		min_samples = 128;
416 
417 	while ((c = getopt_long(argc, argv, "hvu:b:s:t:T:",
418 					options, &option_index)) != -1) {
419 		switch (c) {
420 		case 'h':
421 			usage(MY_NAME, options, options_descriptions);
422 			return EXIT_SUCCESS;
423 		case 'u':
424 			info.arg_index += 2;
425 			info.uri_index = info.arg_index;
426 			break;
427 		case 'b':
428 			info.arg_index += 2;
429 			/* Max 4M , min 64 bytes (cache line) */
430 			info.buffer_size = sanitize_clamp("buffersize", info.argv[info.arg_index],
431 					min_samples, 1024 * 1024 * 4);
432 			break;
433 		case 't':
434 			info.arg_index +=2;
435 			/* ensure between least once a day and never (0) */
436 			info.timeout = 1000 * sanitize_clamp("timeout", info.argv[info.arg_index],
437 					0, 60 * 60 * 24);
438 			break;
439 		case 'T':
440 			info.arg_index +=2;
441 			/* Max number threads 1024, min 1 */
442 			info.num_threads = sanitize_clamp("threads", info.argv[info.arg_index],
443 					1, 1024);
444 			break;
445 		case 'v':
446 			if (!info.verbose)
447 				info.arg_index++;
448 			info.verbose++;
449 			break;
450 		case '?':
451 			return EXIT_FAILURE;
452 		}
453 	}
454 
455 	if (info.arg_index + 1 >= argc) {
456 		fprintf(stderr, "Incorrect number of arguments.\n");
457 		if (info.uri_index) {
458 			struct iio_context *ctx = iio_create_context_from_uri(info.argv[info.uri_index]);
459 			if (ctx) {
460 				fprintf(stderr, "checking uri %s\n", info.argv[info.uri_index]);
461 				i = iio_context_set_timeout(ctx, 500);
462 				thread_err(-1, i, "iio_context_set_timeout fail");
463 				unsigned int nb_devices = iio_context_get_devices_count(ctx);
464 				for (i = 0; i < nb_devices; i++) {
465 					unsigned int j;
466 					const struct iio_device *dev = iio_context_get_device(ctx, i);
467 					const char *name = iio_device_get_name(dev);
468 					unsigned int nb_channels = iio_device_get_channels_count(dev);
469 					if (!iio_device_get_buffer_attrs_count(dev))
470 						continue;
471 					for (j = 0; j < nb_channels; j++) {
472 						struct iio_channel *ch = iio_device_get_channel(dev, j);
473 						if (iio_channel_is_output(ch))
474 							continue;
475 						iio_channel_enable(ch);
476 					}
477 					struct iio_buffer *buffer = iio_device_create_buffer(dev, info.buffer_size, false);
478 					if (buffer) {
479 						iio_buffer_destroy(buffer);
480 						printf("try : %s\n", name);
481 					}
482 				}
483 				iio_context_destroy(ctx);
484 			} else {
485 				fprintf(stderr, "need valid uri\n");
486 			}
487 		}
488 		fprintf(stderr, "\n");
489 		usage(MY_NAME, options, options_descriptions);
490 		return EXIT_FAILURE;
491 	}
492 
493 	if (info.uri_index) {
494 		struct iio_context *ctx = iio_create_context_from_uri(info.argv[info.uri_index]);
495 		if (!ctx) {
496 			fprintf(stderr, "need valid uri\n");
497 			usage(MY_NAME, options, options_descriptions);
498 			return EXIT_FAILURE;
499 		}
500 		iio_context_destroy(ctx);
501 		if (!strncmp(info.argv[info.uri_index], "usb:", strlen("usb:")))
502 			info.back = IIO_USB;
503 		else if (!strncmp(info.argv[info.uri_index], "ip:", strlen("ip:")))
504 			info.back = IIO_NETWORK;
505 		else if (!strncmp(info.argv[info.uri_index], "local:", strlen("local:")))
506 			info.back = IIO_LOCAL;
507 
508 	} else {
509 		fprintf(stderr, "need valid uri\n");
510 		usage(MY_NAME, options, options_descriptions);
511 		return EXIT_FAILURE;
512 	}
513 
514 	/* prep memory for all the threads */
515 	size_t histogram[10];
516 	histogram[0] = histogram[1] = histogram[2] = histogram[3] = histogram[4] = 0;
517 	histogram[5] = histogram[6] = histogram[7] = histogram[8] = 0;
518 
519 	info.threads = calloc(info.num_threads, sizeof(*info.threads));
520 	info.tid = calloc(info.num_threads, sizeof(*info.threads));
521 	info.starts = calloc(info.num_threads, sizeof(unsigned int));
522 	info.buffers = calloc(info.num_threads, sizeof(unsigned int));
523 	info.refills = calloc(info.num_threads, sizeof(unsigned int));
524 	info.start = calloc(info.num_threads, sizeof(struct timeval *));
525 
526 	ret = (void *)calloc(info.num_threads, sizeof(void *));
527 
528 	for (i = 0; i < info.num_threads; i++) {
529 		info.start[i] = malloc(NUM_TIMESTAMPS * sizeof(struct timeval));
530 	}
531 
532 	sigfillset(&set);
533 	/* turn off buffering */
534 	setbuf(stdout, NULL);
535 
536 	gettimeofday(&s_loop, NULL);
537 	while (app_running) {
538 		unsigned int flag;
539 
540 		/* start all the threads */
541 		threads_running = true;
542 		pthread_sigmask(SIG_BLOCK, &set, &oldset);
543 		for (i = 0; i < info.num_threads; i++) {
544 			/* before starting a thread, set up things */
545 			info.start[i][0].tv_sec = 0; info.start[i][0].tv_usec = 0;
546 			memset(&info.tid[i], -1, sizeof(pthread_t));
547 			pthread_create(&info.threads[i], NULL, client_thread, &info);
548 		}
549 		pthread_sigmask(SIG_SETMASK, &oldset, NULL);
550 		gettimeofday(&start, NULL);
551 
552 		/* If a thread prematurely dies, start it again */
553 		while (app_running && threads_running) {
554 			/* If we find a thread that isn't running, restart it */
555 			for (i = 0; i < info.num_threads && threads_running; i++){
556 				if (info.tid[i] == 0){
557 					if (info.verbose == VERYVERBOSE)
558 						printf("waiting for %u\n", i);
559 					pret = pthread_join(info.threads[i], &ret[i]);
560 					thread_err(-1, pret, "pthread_join fail");
561 					if (pret < 0) {
562 						app_running = 0;
563 					} else {
564 						memset(&info.tid[i], -1, sizeof(pthread_t));
565 						pthread_create(&info.threads[i], NULL, client_thread, &info);
566 					}
567 				}
568 			}
569 
570 			/* if we timeout, stop */
571 			gettimeofday(&end, NULL);
572 			duration = ((end.tv_sec - start.tv_sec) * 1000) +
573 					((end.tv_usec - start.tv_usec) / 1000);
574 			if (info.timeout && duration >= info.timeout) {
575 				threads_running = false;
576 			} else {
577 				struct timespec wait;
578 				wait.tv_sec = 0;
579 				wait.tv_nsec = (1000 * 1000);
580 				nanosleep(&wait, &wait);
581 			}
582 		}
583 
584 		gettimeofday(&end, NULL);
585 		duration = ((end.tv_sec - s_loop.tv_sec) * 1000) +
586 			((end.tv_usec - s_loop.tv_usec) / 1000);
587 
588 		flag = 0;
589 		threads_running = false;
590 
591 		/* let all the threads end */
592 		if (!app_running || info.verbose >= SUMMARY)
593 			printf("-------------------------------------------------------------\n");
594 		for (i = 0; i < info.num_threads; i++) {
595 			pret = pthread_join(info.threads[i], &ret[i]);
596 			thread_err(-1, pret, "pthread_join fail");
597 			if (pret < 0)
598 				app_running = 0;
599 		}
600 		/* Did at least one thread end in success? */
601 		for (i = 0; i < info.num_threads; i++) {
602 			if (!((int) (intptr_t)ret[i])) {
603 				flag = 1;
604 				break;
605 			}
606 		}
607 		if (!flag) {
608 			app_running = 0;
609 			printf("All threads failed\n");
610 		}
611 
612 		/* Calculate some stats about the threads */
613 		unsigned int a =0, b = 0;
614 		c = 0;
615 		for (i = 0; i < info.num_threads; i++) {
616 			a+= info.starts[i];
617 			b+= info.buffers[i];
618 			c+= info.refills[i];
619 			if (!app_running || info.verbose >= VERBOSE)
620 				printf("%2u: Ran : %u times, opening %u buffers, doing %u refills\n",
621 						i, info.starts[i], info.buffers[i], info.refills[i]);
622 		}
623 		if (!app_running || info.verbose >= SUMMARY)
624 			printf("total: ");
625 		i = duration/1000;
626 		flag=0;
627 		if (i > 60*60*24) {
628 			if (!app_running || info.verbose >= SUMMARY)
629 				printf("%ud", i/(60*60*24));
630 			i -= (i/(60*60*24))*60*60*24;
631 			flag = 1;
632 		}
633 		if (flag || i > 60*60) {
634 			if (!app_running || info.verbose >= SUMMARY) {
635 				if (flag)
636 					printf("%02uh", i/(60*60));
637 				else
638 					printf("%uh", i/(60*60));
639 			}
640 			i -= (i/(60*60))*60*60;
641 			flag = 1;
642 		}
643 		if (flag || i > 60) {
644 			if (!app_running || info.verbose >= SUMMARY) {
645 				if (flag)
646 					printf("%02um", i/60);
647 				else
648 					printf("%um", i/60);
649 			}
650 			i -= (i/60)*60;
651 			flag = 1;
652 		}
653 		if (flag || i) {
654 			if (!app_running || info.verbose >= SUMMARY)
655 				printf("%02us", i);
656 		}
657 
658 		if (!app_running || info.verbose >= SUMMARY) {
659 			printf(" Context : %i (%2.2f/s), buffers: %i (%2.2f/s), refills : %i (%2.2f/s)\n",
660 					a, (double)a * 1000 / duration,
661 					b, (double)b * 1000 / duration,
662 					c, (double)c * 1000 / duration);
663 		}
664 		/* gather and sort things, so we can print out a histogram */
665 		struct timeval *sort;
666 		sort = calloc(info.num_threads * NUM_TIMESTAMPS, sizeof(struct timeval));
667 		b = 0;
668 		/* gather */
669 		for (i = 0; i < info.num_threads; i++) {
670 			for (a = 0; a < NUM_TIMESTAMPS; a++) {
671 				if (info.start[i][a].tv_sec) {
672 					sort[b].tv_sec = info.start[i][a].tv_sec;
673 					sort[b].tv_usec = info.start[i][a].tv_usec;
674 					b++;
675 				} else {
676 					/* if we hit a zero, this loop is done */
677 					break;
678 				}
679 			}
680 		}
681 		/* sort */
682 		qsort(sort, b, sizeof(struct timeval), compare_timeval);
683 		/* bin */
684 		for (i = 1; i < b; i++) {
685 			duration = (sort[i].tv_sec - sort[i -1].tv_sec) * 1000000 +
686 				sort[i].tv_usec - sort[i - 1].tv_usec;
687 			histogram[8]++;
688 			if (duration == 0)
689 				histogram[0]++;
690 			else if (duration < 10)
691 				histogram[1]++;
692 			else if (duration < 100)
693 				histogram[2]++;
694 			else if (duration < 1000)
695 				histogram[3]++;
696 			else if (duration < 10000)
697 				histogram[4]++;
698 			else if (duration < 100000)
699 				histogram[5]++;
700 			else if (duration < 1000000)
701 				histogram[6]++;
702 			else
703 				histogram[7]++;
704 		}
705 		/* dump */
706 		if (!app_running || info.verbose >= SUMMARY) {
707 			printf("    0        : %7zu (%5.2f%%)\n",
708 					histogram[0],
709 					(double)histogram[0]*100/histogram[8]);
710 			printf("  1 - 9   μs : %7zu (%5.2f%%)\n",
711 					histogram[1],
712 					(double)histogram[1]*100/histogram[8]);
713 			printf(" 10 - 99  μs : %7zu (%5.2f%%)\n",
714 					histogram[2],
715 					(double)histogram[2]*100/histogram[8]);
716 			printf("100 - 999 μs : %7zu (%5.2f%%)\n",
717 					histogram[3],
718 					(double)histogram[3]*100/histogram[8]);
719 			printf("  1 - 9.9 ms : %7zu (%5.2f%%)\n",
720 					histogram[4],
721 					(double)histogram[4]*100/histogram[8]);
722 			printf(" 10 - 99  ms : %7zu (%5.2f%%)\n",
723 					histogram[5],
724 					(double)histogram[5]*100/histogram[8]);
725 			printf("100 - 999 ms : %7zu (%5.2f%%)\n",
726 					histogram[6],
727 					(double)histogram[6]*100/histogram[8]);
728 			printf("over 1 s     : %7zu (%5.2f%%)\n",
729 					histogram[7],
730 					(double)histogram[7]*100/histogram[8]);
731 			printf("\n");
732 		}
733 		free(sort);
734 
735 		/* if the app is still running, go again */
736 	}
737 
738 	free(info.threads);
739 	free(info.tid);
740 	free(info.starts);
741 	free(info.buffers);
742 	free(info.refills);
743 	for (i = 0; i < info.num_threads; i++)
744 		free(info.start[i]);
745 	free(info.start);
746 	free(ret);
747 	return 0;
748 }
749