1 /*
2
3 ----------------------------------------------------
4 httpry - HTTP logging and information retrieval tool
5 ----------------------------------------------------
6
7 Copyright (c) 2005-2014 Jason Bittel <jason.bittel@gmail.com>
8
9 */
10
11 #include <math.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <pthread.h>
16 #include <time.h>
17 #include <unistd.h>
18 #include "config.h"
19 #include "error.h"
20 #include "rate.h"
21 #include "utility.h"
22
23 #define MAX_HOST_LEN 255
24 #define HASHSIZE 2048
25 #define NODE_BLOCKSIZE 100
26 #define NODE_ALLOC_BLOCKSIZE 10
27
28 struct host_stats {
29 char host[MAX_HOST_LEN + 1];
30 unsigned int count;
31 time_t first_packet;
32 time_t last_packet;
33 struct host_stats *next;
34 };
35
36 struct thread_args {
37 char *use_infile;
38 unsigned int rate_interval;
39 int rate_threshold;
40 };
41
42 void create_rate_stats_thread(int rate_interval, char *use_infile, int rate_threshold);
43 void exit_rate_stats_thread();
44 void *run_stats(void *args);
45 struct host_stats *remove_node(struct host_stats *node, struct host_stats *prev);
46 struct host_stats *get_host(char *str);
47 struct host_stats *get_node();
48
49 static pthread_t thread;
50 static int thread_created = 0;
51 static pthread_mutex_t stats_lock;
52 static struct host_stats **stats = NULL;
53 static struct host_stats *free_stack = NULL;
54 static struct host_stats **block_alloc = NULL;
55 static struct host_stats totals;
56 static struct thread_args thread_args;
57
58 /* Initialize rate stats counters and structures, and
59 start up the stats thread if necessary */
init_rate_stats(int rate_interval,char * use_infile,int rate_threshold)60 void init_rate_stats(int rate_interval, char *use_infile, int rate_threshold) {
61 /* Initialize host totals */
62 totals.count = 0;
63 totals.first_packet = 0;
64 totals.last_packet = 0;
65
66 /* Allocate host stats hash array */
67 if ((stats = (struct host_stats **) calloc(HASHSIZE, sizeof(struct host_stats *))) == NULL)
68 LOG_DIE("Cannot allocate memory for host stats");
69
70 if (!use_infile)
71 create_rate_stats_thread(rate_interval, use_infile, rate_threshold);
72
73 return;
74 }
75
76 /* Spawn a thread for updating and printing rate statistics */
create_rate_stats_thread(int rate_interval,char * use_infile,int rate_threshold)77 void create_rate_stats_thread(int rate_interval, char *use_infile, int rate_threshold) {
78 sigset_t set;
79 int s;
80
81 if (thread_created) return;
82
83 thread_args.use_infile = use_infile;
84 thread_args.rate_interval = rate_interval;
85 thread_args.rate_threshold = rate_threshold;
86
87 sigemptyset(&set);
88 sigaddset(&set, SIGINT);
89 sigaddset(&set, SIGHUP);
90
91 s = pthread_mutex_init(&stats_lock, NULL);
92 if (s != 0)
93 LOG_DIE("Statistics thread mutex initialization failed with error %d", s);
94
95 s = pthread_sigmask(SIG_BLOCK, &set, NULL);
96 if (s != 0)
97 LOG_DIE("Statistics thread signal blocking failed with error %d", s);
98
99 s = pthread_create(&thread, NULL, run_stats, (void *) &thread_args);
100 if (s != 0)
101 LOG_DIE("Statistics thread creation failed with error %d", s);
102
103 s = pthread_sigmask(SIG_UNBLOCK, &set, NULL);
104 if (s != 0)
105 LOG_DIE("Statistics thread signal unblocking failed with error %d", s);
106
107 thread_created = 1;
108
109 return;
110 }
111
112 /* Attempt to cancel the stats thread, cleanup allocated
113 memory and clear necessary counters and structures */
cleanup_rate_stats()114 void cleanup_rate_stats() {
115 struct host_stats **i;
116
117 exit_rate_stats_thread();
118
119 if (block_alloc != NULL) {
120 for (i = block_alloc; *i; i++) {
121 free(*i);
122 }
123
124 free(block_alloc);
125 block_alloc = NULL;
126 }
127
128 if (stats != NULL) {
129 free(stats);
130 stats = NULL;
131 }
132
133 free_stack = NULL;
134
135 return;
136 }
137
138 /* Explicitly exit rate statistics thread */
exit_rate_stats_thread()139 void exit_rate_stats_thread() {
140 int s;
141 void *retval;
142
143 if (!thread_created) return;
144
145 s = pthread_cancel(thread);
146 if (s != 0)
147 LOG_WARN("Statistics thread cancellation failed with error %d", s);
148
149 s = pthread_join(thread, &retval);
150 if (s != 0)
151 LOG_WARN("Statistics thread join failed with error %d", s);
152
153 if (retval != PTHREAD_CANCELED)
154 LOG_WARN("Statistics thread exit value was unexpected");
155
156 thread_created = 0;
157
158 s = pthread_mutex_destroy(&stats_lock);
159 if (s != 0)
160 LOG_WARN("Statistcs thread mutex destroy failed with error %d", s);
161
162 return;
163 }
164
165 /* This is our statistics thread */
run_stats(void * args)166 void *run_stats (void *args) {
167 struct thread_args *thread_args = (struct thread_args *) args;
168
169 while (1) {
170 sleep(thread_args->rate_interval);
171 display_rate_stats(thread_args->use_infile, thread_args->rate_threshold);
172 }
173
174 return (void *) 0;
175 }
176
177 /* Display the running average within each valid stats node */
display_rate_stats(char * use_infile,int rate_threshold)178 void display_rate_stats(char *use_infile, int rate_threshold) {
179 time_t now;
180 char st_time[MAX_TIME_LEN];
181 unsigned int delta, rps = 0;
182 int i;
183 struct host_stats *node, *prev;
184
185 if (stats == NULL) return;
186
187 if (thread_created)
188 pthread_mutex_lock(&stats_lock);
189
190 if (use_infile) {
191 now = totals.last_packet;
192 } else {
193 now = time(NULL);
194 }
195
196 strftime(st_time, MAX_TIME_LEN, "%Y-%m-%d %H:%M:%S", localtime(&now));
197
198 #ifdef DEBUG
199 int j, num_buckets = 0, num_chain, max_chain = 0, num_nodes = 0;
200
201 for (j = 0; j < HASHSIZE; j++) {
202 if (stats[j]) num_buckets++;
203
204 num_chain = 0;
205 for (node = stats[j]; node != NULL; node = node->next) num_chain++;
206 if (num_chain > max_chain) max_chain = num_chain;
207 num_nodes += num_chain;
208 }
209
210 PRINT("----------------------------");
211 PRINT("Hash buckets: %d", HASHSIZE);
212 PRINT("Nodes inserted: %d", num_nodes);
213 PRINT("Buckets in use: %d", num_buckets);
214 PRINT("Hash collisions: %d", num_nodes - num_buckets);
215 PRINT("Longest hash chain: %d", max_chain);
216 PRINT("----------------------------");
217 #endif
218
219 /* Display rate stats for each valid host */
220 for (i = 0; i < HASHSIZE; i++) {
221 node = stats[i];
222 prev = NULL;
223
224 while (node != NULL) {
225 delta = now - node->first_packet;
226 if (delta > 0) {
227 rps = (unsigned int) ceil(node->count / (float) delta);
228 } else {
229 rps = 0;
230 }
231
232 if (rps >= rate_threshold) {
233 printf("%s%s%s%s%u rps\n", st_time, FIELD_DELIM, node->host, FIELD_DELIM, rps);
234 prev = node;
235 node = node->next;
236 } else {
237 node = remove_node(node, prev);
238 }
239 }
240 }
241
242 /* Display rate totals */
243 delta = (unsigned int) (now - totals.first_packet);
244 if (delta > 0)
245 printf("%s%stotals%s%3.2f rps\n", st_time, FIELD_DELIM, FIELD_DELIM, (float) totals.count / delta);
246
247 if (thread_created)
248 pthread_mutex_unlock(&stats_lock);
249
250 return;
251 }
252
253 /* Remove the given node from the hash and return it to the free stack;
254 returns the correct node for continuing to traverse the hash */
remove_node(struct host_stats * node,struct host_stats * prev)255 struct host_stats *remove_node(struct host_stats *node, struct host_stats *prev) {
256 struct host_stats *next;
257 unsigned int hashval;
258
259 /* Unlink the node from the hash */
260 if (prev == NULL) {
261 hashval = hash_str(node->host, HASHSIZE);
262
263 if (node->next) {
264 stats[hashval] = node->next;
265 } else {
266 stats[hashval] = NULL;
267 }
268 next = stats[hashval];
269 } else {
270 if (node->next) {
271 prev->next = node->next;
272 } else {
273 prev->next = NULL;
274 }
275 next = prev->next;
276 }
277
278 /* Add the node to the head of the free stack */
279 node->next = free_stack;
280 free_stack = node;
281
282 return next;
283 }
284
285 /* Update the stats for a given host; if the host is not
286 found in the hash, add it */
update_host_stats(char * host,time_t t)287 void update_host_stats(char *host, time_t t) {
288 struct host_stats *node;
289 unsigned int hashval;
290
291 if ((host == NULL) || (stats == NULL)) return;
292
293 if (thread_created)
294 pthread_mutex_lock(&stats_lock);
295
296 if ((node = get_host(host)) == NULL) {
297 node = get_node();
298
299 hashval = hash_str(host, HASHSIZE);
300
301 #ifdef DEBUG
302 ASSERT((hashval >= 0) && (hashval < HASHSIZE));
303 #endif
304
305 str_copy(node->host, host, MAX_HOST_LEN);
306 node->count = 0;
307 node->first_packet = t;
308
309 /* Link node into hash */
310 node->next = stats[hashval];
311 stats[hashval] = node;
312 }
313
314 if (node->first_packet == 0)
315 node->first_packet = t;
316 node->last_packet = t;
317 node->count++;
318
319 if (totals.first_packet == 0)
320 totals.first_packet = t;
321 totals.last_packet = t;
322 totals.count++;
323
324 if (thread_created)
325 pthread_mutex_unlock(&stats_lock);
326
327 return;
328 }
329
330 /* Lookup a particular node in hash; return pointer to node
331 if found, NULL otherwise */
get_host(char * str)332 struct host_stats *get_host(char *str) {
333 struct host_stats *node;
334
335 #ifdef DEBUG
336 ASSERT(str);
337 ASSERT(strlen(str) > 0);
338 ASSERT((hash_str(str, HASHSIZE) >= 0) && (hash_str(str, HASHSIZE) < HASHSIZE));
339 #endif
340
341 for (node = stats[hash_str(str, HASHSIZE)]; node != NULL; node = node->next)
342 if (str_compare(str, node->host) == 0)
343 return node;
344
345 return NULL;
346 }
347
348 /* Get a new node from either the free stack or an allocated block;
349 if the block is empty, allocate a new chunk of memory */
get_node()350 struct host_stats *get_node() {
351 static struct host_stats *block, *tail, **mv;
352 struct host_stats *head, **tmp;
353 static int alloc_size;
354
355 /* Initialize static variables as necessary */
356 if (block_alloc == NULL) {
357 block = NULL;
358 alloc_size = 0;
359 }
360
361 if (free_stack != NULL) { /* Get node from free stack */
362 head = free_stack;
363 free_stack = free_stack->next;
364 head->next = NULL;
365 } else if (block != NULL) { /* Get node from allocated block */
366 head = block;
367 if (block == tail) {
368 block = NULL;
369 } else {
370 block++;
371 }
372 } else { /* Out of nodes, allocate a new block */
373 if ((block = (struct host_stats *) malloc(NODE_BLOCKSIZE * sizeof(struct host_stats))) == NULL) {
374 LOG_DIE("Cannot allocate memory for node block");
375 }
376
377 /* Store pointer to allocated block so we can free it later */
378 if (block_alloc == NULL) {
379 if ((block_alloc = (struct host_stats **) malloc(NODE_ALLOC_BLOCKSIZE * sizeof(struct host_stats *))) == NULL) {
380 LOG_DIE("Cannot allocate memory for blocks array");
381 }
382
383 mv = block_alloc;
384 }
385
386 *mv = block;
387
388 if (++alloc_size % NODE_ALLOC_BLOCKSIZE == 0) {
389 tmp = realloc(block_alloc, ((alloc_size + NODE_ALLOC_BLOCKSIZE) * sizeof(struct host_stats *)));
390 if (tmp == NULL) {
391 LOG_DIE("Cannot re-allocate memory for blocks array");
392 }
393 block_alloc = tmp;
394 mv = block_alloc + alloc_size - 1;
395 }
396
397 mv++;
398 *mv = NULL;
399
400 tail = block + NODE_BLOCKSIZE - 1;
401 head = block;
402 block++;
403 }
404
405 return head;
406 }
407