1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "decode.h"
29 #include "detect.h"
30 #include "detect-engine.h"
31 #include "conf.h"
32 
33 #include "tm-threads.h"
34 
35 #include "util-unittest.h"
36 #include "util-byte.h"
37 #include "util-profiling.h"
38 #include "util-profiling-locks.h"
39 
40 #ifdef PROFILING
41 
42 #ifndef MIN
43 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
44 #endif
45 
46 /**
47  * Extra data for rule profiling.
48  */
49 typedef struct SCProfileKeywordData_ {
50     uint64_t checks;
51     uint64_t matches;
52     uint64_t max;
53     uint64_t ticks_match;
54     uint64_t ticks_no_match;
55 } SCProfileKeywordData;
56 
57 typedef struct SCProfileKeywordDetectCtx_ {
58     uint32_t id;
59     SCProfileKeywordData *data;
60     pthread_mutex_t data_m;
61 } SCProfileKeywordDetectCtx;
62 
63 static int profiling_keywords_output_to_file = 0;
64 int profiling_keyword_enabled = 0;
65 thread_local int profiling_keyword_entered = 0;
66 static char profiling_file_name[PATH_MAX];
67 static const char *profiling_file_mode = "a";
68 
SCProfilingKeywordsGlobalInit(void)69 void SCProfilingKeywordsGlobalInit(void)
70 {
71     ConfNode *conf;
72 
73     conf = ConfGetNode("profiling.keywords");
74     if (conf != NULL) {
75         if (ConfNodeChildValueIsTrue(conf, "enabled")) {
76             profiling_keyword_enabled = 1;
77             const char *filename = ConfNodeLookupChildValue(conf, "filename");
78             if (filename != NULL) {
79                 const char *log_dir;
80                 log_dir = ConfigGetLogDirectory();
81 
82                 snprintf(profiling_file_name, sizeof(profiling_file_name), "%s/%s",
83                         log_dir, filename);
84 
85                 const char *v = ConfNodeLookupChildValue(conf, "append");
86                 if (v == NULL || ConfValIsTrue(v)) {
87                     profiling_file_mode = "a";
88                 } else {
89                     profiling_file_mode = "w";
90                 }
91 
92                 profiling_keywords_output_to_file = 1;
93             }
94         }
95     }
96 }
97 
DoDump(SCProfileKeywordDetectCtx * rules_ctx,FILE * fp,const char * name)98 static void DoDump(SCProfileKeywordDetectCtx *rules_ctx, FILE *fp, const char *name)
99 {
100     int i;
101     fprintf(fp, "  ----------------------------------------------"
102             "------------------------------------------------------"
103             "----------------------------\n");
104     fprintf(fp, "  Stats for: %s\n", name);
105     fprintf(fp, "  ----------------------------------------------"
106             "------------------------------------------------------"
107             "----------------------------\n");
108     fprintf(fp, "  %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Keyword", "Ticks", "Checks", "Matches", "Max Ticks", "Avg", "Avg Match", "Avg No Match");
109     fprintf(fp, "  ---------------- "
110                 "--------------- "
111                 "--------------- "
112                 "--------------- "
113                 "--------------- "
114                 "--------------- "
115                 "--------------- "
116                 "--------------- "
117         "\n");
118     for (i = 0; i < DETECT_TBLSIZE; i++) {
119         SCProfileKeywordData *d = &rules_ctx->data[i];
120         if (d == NULL || d->checks == 0)
121             continue;
122 
123         uint64_t ticks = d->ticks_match + d->ticks_no_match;
124         double avgticks = 0;
125         double avgticks_match = 0;
126         double avgticks_no_match = 0;
127         if (ticks && d->checks) {
128             avgticks = (ticks / d->checks);
129 
130             if (d->ticks_match && d->matches)
131                 avgticks_match = (d->ticks_match / d->matches);
132             if (d->ticks_no_match && (d->checks - d->matches) != 0)
133                 avgticks_no_match = (d->ticks_no_match / (d->checks - d->matches));
134         }
135 
136         fprintf(fp,
137             "  %-16s %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15.2f %-15.2f\n",
138             sigmatch_table[i].name,
139             ticks,
140             d->checks,
141             d->matches,
142             d->max,
143             avgticks,
144             avgticks_match,
145             avgticks_no_match);
146     }
147 }
148 
149 static void
SCProfilingKeywordDump(DetectEngineCtx * de_ctx)150 SCProfilingKeywordDump(DetectEngineCtx *de_ctx)
151 {
152     int i;
153     FILE *fp;
154     struct timeval tval;
155     struct tm *tms;
156     struct tm local_tm;
157 
158     if (profiling_keyword_enabled == 0)
159         return;
160 
161     const int nlists = de_ctx->buffer_type_id;
162     gettimeofday(&tval, NULL);
163     tms = SCLocalTime(tval.tv_sec, &local_tm);
164 
165     if (profiling_keywords_output_to_file == 1) {
166         SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
167 
168         fp = fopen(profiling_file_name, profiling_file_mode);
169 
170         if (fp == NULL) {
171             SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", profiling_file_name,
172                     strerror(errno));
173             return;
174         }
175     } else {
176        fp = stdout;
177     }
178 
179     fprintf(fp, "  ----------------------------------------------"
180             "------------------------------------------------------"
181             "----------------------------\n");
182     fprintf(fp, "  Date: %" PRId32 "/%" PRId32 "/%04d -- "
183             "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
184             tms->tm_hour,tms->tm_min, tms->tm_sec);
185 
186     /* global stats first */
187     DoDump(de_ctx->profile_keyword_ctx, fp, "total");
188     /* per buffer stats next, but only if there are stats to print */
189     for (i = 0; i < nlists; i++) {
190         int j;
191         uint64_t checks = 0;
192         for (j = 0; j < DETECT_TBLSIZE; j++) {
193             checks += de_ctx->profile_keyword_ctx_per_list[i]->data[j].checks;
194         }
195 
196         if (checks) {
197             const char *name = NULL;
198             if (i < DETECT_SM_LIST_DYNAMIC_START) {
199                 name = DetectSigmatchListEnumToString(i);
200             } else {
201                 name = DetectBufferTypeGetNameById(de_ctx, i);
202             }
203 
204             DoDump(de_ctx->profile_keyword_ctx_per_list[i], fp, name);
205         }
206     }
207 
208     fprintf(fp,"\n");
209     if (fp != stdout)
210         fclose(fp);
211 
212     SCLogPerf("Done dumping keyword profiling data.");
213 }
214 
215 /**
216  * \brief Update a rule counter.
217  *
218  * \param id The ID of this counter.
219  * \param ticks Number of CPU ticks for this rule.
220  * \param match Did the rule match?
221  */
222 void
SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx * det_ctx,int id,uint64_t ticks,int match)223 SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
224 {
225     if (det_ctx != NULL && det_ctx->keyword_perf_data != NULL && id < DETECT_TBLSIZE) {
226         SCProfileKeywordData *p = &det_ctx->keyword_perf_data[id];
227 
228         p->checks++;
229         p->matches += match;
230         if (ticks > p->max)
231             p->max = ticks;
232         if (match == 1)
233             p->ticks_match += ticks;
234         else
235             p->ticks_no_match += ticks;
236 
237         /* store per list (buffer type) as well */
238         if (det_ctx->keyword_perf_list >= 0) {// && det_ctx->keyword_perf_list < DETECT_SM_LIST_MAX) {
239             p = &det_ctx->keyword_perf_data_per_list[det_ctx->keyword_perf_list][id];
240             p->checks++;
241             p->matches += match;
242             if (ticks > p->max)
243                 p->max = ticks;
244             if (match == 1)
245                 p->ticks_match += ticks;
246             else
247                 p->ticks_no_match += ticks;
248         }
249     }
250 }
251 
SCProfilingKeywordInitCtx(void)252 static SCProfileKeywordDetectCtx *SCProfilingKeywordInitCtx(void)
253 {
254     SCProfileKeywordDetectCtx *ctx = SCMalloc(sizeof(SCProfileKeywordDetectCtx));
255     if (ctx != NULL) {
256         memset(ctx, 0x00, sizeof(SCProfileKeywordDetectCtx));
257 
258         if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
259                     FatalError(SC_ERR_FATAL,
260                                "Failed to initialize hash table mutex.");
261         }
262     }
263 
264     return ctx;
265 }
266 
DetroyCtx(SCProfileKeywordDetectCtx * ctx)267 static void DetroyCtx(SCProfileKeywordDetectCtx *ctx)
268 {
269     if (ctx) {
270         if (ctx->data != NULL)
271             SCFree(ctx->data);
272         pthread_mutex_destroy(&ctx->data_m);
273         SCFree(ctx);
274     }
275 }
276 
SCProfilingKeywordDestroyCtx(DetectEngineCtx * de_ctx)277 void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx)
278 {
279     if (de_ctx != NULL) {
280         SCProfilingKeywordDump(de_ctx);
281 
282         DetroyCtx(de_ctx->profile_keyword_ctx);
283 
284         const int nlists = de_ctx->buffer_type_id;
285         int i;
286         for (i = 0; i < nlists; i++) {
287             DetroyCtx(de_ctx->profile_keyword_ctx_per_list[i]);
288         }
289         SCFree(de_ctx->profile_keyword_ctx_per_list);
290     }
291 }
292 
SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx * ctx,DetectEngineThreadCtx * det_ctx)293 void SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
294 {
295     if (ctx == NULL)
296         return;
297 
298     SCProfileKeywordData *a = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
299     if (a != NULL) {
300         memset(a, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
301         det_ctx->keyword_perf_data = a;
302     }
303 
304     const int nlists = det_ctx->de_ctx->buffer_type_id;
305     det_ctx->keyword_perf_data_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordData *));
306     BUG_ON(det_ctx->keyword_perf_data_per_list == NULL);
307 
308     int i;
309     for (i = 0; i < nlists; i++) {
310         SCProfileKeywordData *b = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
311         if (b != NULL) {
312             memset(b, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
313             det_ctx->keyword_perf_data_per_list[i] = b;
314         }
315 
316     }
317 }
318 
SCProfilingKeywordThreadMerge(DetectEngineCtx * de_ctx,DetectEngineThreadCtx * det_ctx)319 static void SCProfilingKeywordThreadMerge(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
320 {
321     if (de_ctx == NULL || de_ctx->profile_keyword_ctx == NULL ||
322         de_ctx->profile_keyword_ctx->data == NULL || det_ctx == NULL ||
323         det_ctx->keyword_perf_data == NULL)
324         return;
325 
326     int i;
327     for (i = 0; i < DETECT_TBLSIZE; i++) {
328         de_ctx->profile_keyword_ctx->data[i].checks += det_ctx->keyword_perf_data[i].checks;
329         de_ctx->profile_keyword_ctx->data[i].matches += det_ctx->keyword_perf_data[i].matches;
330         de_ctx->profile_keyword_ctx->data[i].ticks_match += det_ctx->keyword_perf_data[i].ticks_match;
331         de_ctx->profile_keyword_ctx->data[i].ticks_no_match += det_ctx->keyword_perf_data[i].ticks_no_match;
332         if (det_ctx->keyword_perf_data[i].max > de_ctx->profile_keyword_ctx->data[i].max)
333             de_ctx->profile_keyword_ctx->data[i].max = det_ctx->keyword_perf_data[i].max;
334     }
335 
336     const int nlists = det_ctx->de_ctx->buffer_type_id;
337     int j;
338     for (j = 0; j < nlists; j++) {
339         for (i = 0; i < DETECT_TBLSIZE; i++) {
340             de_ctx->profile_keyword_ctx_per_list[j]->data[i].checks += det_ctx->keyword_perf_data_per_list[j][i].checks;
341             de_ctx->profile_keyword_ctx_per_list[j]->data[i].matches += det_ctx->keyword_perf_data_per_list[j][i].matches;
342             de_ctx->profile_keyword_ctx_per_list[j]->data[i].ticks_match += det_ctx->keyword_perf_data_per_list[j][i].ticks_match;
343             de_ctx->profile_keyword_ctx_per_list[j]->data[i].ticks_no_match += det_ctx->keyword_perf_data_per_list[j][i].ticks_no_match;
344             if (det_ctx->keyword_perf_data_per_list[j][i].max > de_ctx->profile_keyword_ctx_per_list[j]->data[i].max)
345                 de_ctx->profile_keyword_ctx_per_list[j]->data[i].max = det_ctx->keyword_perf_data_per_list[j][i].max;
346         }
347     }
348 }
349 
SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx * det_ctx)350 void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx)
351 {
352     if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->keyword_perf_data == NULL)
353         return;
354 
355     pthread_mutex_lock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
356     SCProfilingKeywordThreadMerge(det_ctx->de_ctx, det_ctx);
357     pthread_mutex_unlock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
358 
359     SCFree(det_ctx->keyword_perf_data);
360     det_ctx->keyword_perf_data = NULL;
361 
362     const int nlists = det_ctx->de_ctx->buffer_type_id;
363     int i;
364     for (i = 0; i < nlists; i++) {
365         SCFree(det_ctx->keyword_perf_data_per_list[i]);
366         det_ctx->keyword_perf_data_per_list[i] = NULL;
367     }
368     SCFree(det_ctx->keyword_perf_data_per_list);
369 }
370 
371 /**
372  * \brief Register the keyword profiling counters.
373  *
374  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
375  */
376 void
SCProfilingKeywordInitCounters(DetectEngineCtx * de_ctx)377 SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
378 {
379     if (profiling_keyword_enabled == 0)
380         return;
381 
382     const int nlists = de_ctx->buffer_type_id;
383 
384     de_ctx->profile_keyword_ctx = SCProfilingKeywordInitCtx();
385     BUG_ON(de_ctx->profile_keyword_ctx == NULL);
386 
387     de_ctx->profile_keyword_ctx->data = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
388     BUG_ON(de_ctx->profile_keyword_ctx->data == NULL);
389     memset(de_ctx->profile_keyword_ctx->data, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
390 
391     de_ctx->profile_keyword_ctx_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordDetectCtx *));
392     BUG_ON(de_ctx->profile_keyword_ctx_per_list == NULL);
393 
394     int i;
395     for (i = 0; i < nlists; i++) {
396         de_ctx->profile_keyword_ctx_per_list[i] = SCProfilingKeywordInitCtx();
397         BUG_ON(de_ctx->profile_keyword_ctx_per_list[i] == NULL);
398         de_ctx->profile_keyword_ctx_per_list[i]->data = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
399         BUG_ON(de_ctx->profile_keyword_ctx_per_list[i]->data == NULL);
400         memset(de_ctx->profile_keyword_ctx_per_list[i]->data, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
401     }
402 
403     SCLogPerf("Registered %"PRIu32" keyword profiling counters.", DETECT_TBLSIZE);
404 }
405 
406 #endif /* PROFILING */
407