1 /*
2 * Copyright (c) 2013-2020, Peter Haag
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * * Neither the name of the author nor the names of its contributors may be
14 * used to endorse or promote products derived from this software without
15 * specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 *
29 */
30
31 #include "config.h"
32
33 #include <stdio.h>
34 #include <sys/types.h>
35 #include <sys/socket.h>
36 #include <unistd.h>
37 #include <stdlib.h>
38 #include <netinet/in.h>
39 #include <arpa/inet.h>
40 #include <errno.h>
41 #include <string.h>
42 #include <pthread.h>
43
44 #ifdef HAVE_STDINT_H
45 #include <stdint.h>
46 #endif
47
48 #include "util.h"
49 #include "nfdump.h"
50 #include "nffile.h"
51 #include "nfx.h"
52 #include "nfnet.h"
53 #include "output_raw.h"
54 #include "bookkeeper.h"
55 #include "collector.h"
56 #include "exporter.h"
57
58 #include "flowtree.h"
59 #include "netflow_pcap.h"
60
61 extern int verbose;
62 extern extension_descriptor_t extension_descriptor[];
63
64 /* module limited globals */
65 static extension_info_t pcap_extension_info; // common for all pcap records
66 static extension_map_t *pcap_extension_map;
67
68 static uint32_t pcap_output_record_size_v4;
69 static uint32_t pcap_output_record_size_v6;
70
71 typedef struct pcap_v4_block_s {
72 uint32_t srcaddr;
73 uint32_t dstaddr;
74 uint32_t dPkts;
75 uint32_t dOctets;
76 uint32_t data[1]; // link to next record
77 } __attribute__((__packed__ )) pcap_v4_block_t;
78 #define PCAP_V4_BLOCK_DATA_SIZE (sizeof(pcap_v4_block_t) - sizeof(uint32_t))
79
80 typedef struct pcap_v6_block_s {
81 uint64_t srcaddr[2];
82 uint64_t dstaddr[2];
83 uint32_t dPkts;
84 uint32_t dOctets;
85 uint32_t data[1]; // link to next record
86 } __attribute__((__packed__ )) pcap_v6_block_t;
87 #define PCAP_V6_BLOCK_DATA_SIZE (sizeof(pcap_v6_block_t) - sizeof(uint32_t))
88
89 // All required extension to save full pcap records
90 static uint16_t pcap_full_map[] = { EX_LATENCY, 0 };
91
92 #include "nffile_inline.c"
93
Init_pcap2nf(void)94 int Init_pcap2nf(void) {
95 int i, id, map_index;
96 int extension_size;
97 uint16_t map_size;
98
99 // prepare pcap extension map
100 pcap_extension_info.map = NULL;
101 extension_size = 0;
102 map_size = 0;
103
104 i=0;
105 while ( (id = pcap_full_map[i]) != 0 ) {
106 if ( extension_descriptor[id].enabled ) {
107 extension_size += extension_descriptor[id].size;
108 map_size += sizeof(uint16_t);
109 }
110 i++;
111 }
112 // extension_size contains the sum of all optional extensions
113 // caculate the record size
114 pcap_output_record_size_v4 = COMMON_RECORD_DATA_SIZE + PCAP_V4_BLOCK_DATA_SIZE + extension_size;
115 pcap_output_record_size_v6 = COMMON_RECORD_DATA_SIZE + PCAP_V6_BLOCK_DATA_SIZE + extension_size;
116
117 // now the full extension map size
118 map_size += sizeof(extension_map_t);
119
120 // align 32 bits
121 if ( ( map_size & 0x3 ) != 0 )
122 map_size += 2;
123
124 // Create a generic pcap extension map
125 pcap_extension_info.map = (extension_map_t *)malloc((size_t)map_size);
126 if ( !pcap_extension_info.map ) {
127 LogError("Process_pcap: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
128 return 0;
129 }
130 pcap_extension_info.map->type = ExtensionMapType;
131 pcap_extension_info.map->size = map_size;
132 pcap_extension_info.map->map_id = INIT_ID;
133 pcap_extension_info.map->extension_size = extension_size;
134
135 // see netflow_pcap.h for extension map description
136 map_index = 0;
137 i=0;
138 while ( (id = pcap_full_map[i]) != 0 ) {
139 if ( extension_descriptor[id].enabled )
140 pcap_extension_info.map->ex_id[map_index++] = id;
141 i++;
142 }
143 pcap_extension_info.map->ex_id[map_index] = 0;
144
145 pcap_extension_map = NULL;
146
147 return 1;
148
149 } // End of Init_pcap2nf
150
StorePcapFlow(FlowSource_t * fs,struct FlowNode * Node)151 int StorePcapFlow(FlowSource_t *fs, struct FlowNode *Node) {
152 common_record_t *common_record;
153 uint32_t packets, bytes, pcap_output_record_size;
154 uint64_t start_time, end_time;
155 int j, id;
156 char *string;
157 void *data_ptr;
158
159 if ( !pcap_extension_map ) {
160 pcap_extension_map = (extension_map_t *)malloc(pcap_extension_info.map->size);
161 if ( !pcap_extension_map ) {
162 LogError("Process_pcap: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
163 return 0;
164 }
165 memcpy((void *)pcap_extension_map, (void *)pcap_extension_info.map, pcap_extension_info.map->size);
166 if ( !AddExtensionMap(fs, pcap_extension_map) ) {
167 LogError("Process_pcap: Fatal: AddExtensionMap() failed in %s line %d\n", __FILE__, __LINE__);
168 return 0;
169 }
170
171 }
172
173 if ( Node->version == AF_INET6 ) {
174 pcap_output_record_size = pcap_output_record_size_v6;
175 dbg_printf("Store Flow v6 node: size: %u\n", pcap_output_record_size);
176 } else if ( Node->version == AF_INET ) {
177 pcap_output_record_size = pcap_output_record_size_v4;
178 dbg_printf("Store Flow v4 node: size: %u\n", pcap_output_record_size);
179 } else {
180 LogError("Process_pcap: Unexpected version in %s line %d: %u\n", __FILE__, __LINE__, Node->version);
181 return 0;
182 }
183
184 // output buffer size check for all expected records
185 if ( !CheckBufferSpace(fs->nffile, pcap_output_record_size) ) {
186 // fishy! - should never happen. maybe disk full?
187 LogError("Process_pcap: output buffer size error. Abort pcap record processing");
188 return 0;
189 }
190
191 // map output record to memory buffer
192 common_record = (common_record_t *)fs->nffile->buff_ptr;
193
194 // header data
195 common_record->flags = 0;
196 common_record->type = CommonRecordType;
197 common_record->exporter_sysid = 0;
198 common_record->ext_map = pcap_extension_map->map_id;
199 common_record->size = pcap_output_record_size;
200
201 // pcap common fields
202 common_record->srcport = Node->src_port;
203 common_record->dstport = Node->dst_port;
204 common_record->tcp_flags = Node->flags;
205 common_record->prot = Node->proto;
206 common_record->tos = 0;
207 common_record->fwd_status = 0;
208
209 if ( Node->version == AF_INET6 ) {
210 SetFlag(common_record->flags, FLAG_IPV6_ADDR);
211 pcap_v6_block_t *pcap_v6_block = (pcap_v6_block_t *)common_record->data;
212 pcap_v6_block->srcaddr[0] = Node->src_addr.v6[0];
213 pcap_v6_block->srcaddr[1] = Node->src_addr.v6[1];
214 pcap_v6_block->dstaddr[0] = Node->dst_addr.v6[0];
215 pcap_v6_block->dstaddr[1] = Node->dst_addr.v6[1];
216 pcap_v6_block->dPkts = packets = Node->packets;
217 pcap_v6_block->dOctets = bytes = Node->bytes;
218
219 data_ptr = (void *)pcap_v6_block->data;
220 } else {
221 pcap_v4_block_t *pcap_v4_block = (pcap_v4_block_t *)common_record->data;
222 pcap_v4_block->srcaddr = Node->src_addr.v4;
223 pcap_v4_block->dstaddr = Node->dst_addr.v4;
224 pcap_v4_block->dPkts = packets = Node->packets;
225 pcap_v4_block->dOctets = bytes = Node->bytes;
226
227 data_ptr = (void *)pcap_v4_block->data;
228 }
229
230 // process optional extensions
231 j = 0;
232 while ( (id = pcap_extension_map->ex_id[j]) != 0 ) {
233 switch (id) {
234 case EX_IO_SNMP_2: { // 2 byte input/output interface index
235 tpl_ext_4_t *tpl = (tpl_ext_4_t *)data_ptr;
236 tpl->input = 0;
237 tpl->output = 0;
238 data_ptr = (void *)tpl->data;
239 } break;
240 case EX_LATENCY: { // latecy extension
241 tpl_ext_latency_t *tpl = (tpl_ext_latency_t *)data_ptr;
242 tpl->client_nw_delay_usec = Node->latency.client;
243 tpl->server_nw_delay_usec = Node->latency.server;
244 tpl->appl_latency_usec = Node->latency.application;
245 data_ptr = (void *)tpl->data;
246 } break;
247 default:
248 // this should never happen, as pcap has no other extensions
249 LogError("Process_pcap: Unexpected extension %i for pcap record. Skip extension", id);
250 }
251 j++;
252 }
253
254 common_record->first = Node->t_first.tv_sec;
255 common_record->msec_first = Node->t_first.tv_usec / 1000;
256
257 common_record->last = Node->t_last.tv_sec;
258 common_record->msec_last = Node->t_last.tv_usec / 1000;
259
260 start_time = (1000LL * (uint64_t)common_record->first) + (uint64_t)common_record->msec_first;
261 end_time = (1000LL * (uint64_t)common_record->last) + (uint64_t)common_record->msec_last;
262
263 // update first_seen, last_seen
264 if ( start_time < fs->first_seen )
265 fs->first_seen = start_time;
266 if ( end_time > fs->last_seen )
267 fs->last_seen = end_time;
268
269
270 // Update stats
271 switch (common_record->prot) {
272 case IPPROTO_ICMP:
273 case IPPROTO_ICMPV6:
274 fs->nffile->stat_record->numflows_icmp++;
275 fs->nffile->stat_record->numpackets_icmp += packets;
276 fs->nffile->stat_record->numbytes_icmp += bytes;
277 // fix odd CISCO behaviour for ICMP port/type in src port
278 if ( common_record->srcport != 0 ) {
279 uint8_t *s1, *s2;
280 s1 = (uint8_t *)&(common_record->srcport);
281 s2 = (uint8_t *)&(common_record->dstport);
282 s2[0] = s1[1];
283 s2[1] = s1[0];
284 common_record->srcport = 0;
285 }
286 break;
287 case IPPROTO_TCP:
288 fs->nffile->stat_record->numflows_tcp++;
289 fs->nffile->stat_record->numpackets_tcp += packets;
290 fs->nffile->stat_record->numbytes_tcp += bytes;
291 break;
292 case IPPROTO_UDP:
293 fs->nffile->stat_record->numflows_udp++;
294 fs->nffile->stat_record->numpackets_udp += packets;
295 fs->nffile->stat_record->numbytes_udp += bytes;
296 break;
297 default:
298 fs->nffile->stat_record->numflows_other++;
299 fs->nffile->stat_record->numpackets_other += packets;
300 fs->nffile->stat_record->numbytes_other += bytes;
301 }
302
303 fs->nffile->stat_record->numflows++;
304 fs->nffile->stat_record->numpackets += packets;
305 fs->nffile->stat_record->numbytes += bytes;
306
307 if ( verbose ) {
308 master_record_t master_record;
309 ExpandRecord_v2((common_record_t *)common_record, &pcap_extension_info, NULL, &master_record);
310 flow_record_to_raw(&master_record, &string, 0);
311 printf("%s\n", string);
312 }
313
314 // update file record size ( -> output buffer size )
315 fs->nffile->block_header->NumRecords += 1;
316 fs->nffile->block_header->size += pcap_output_record_size;
317 fs->nffile->buff_ptr = data_ptr;
318
319 return 1;
320
321 } /* End of StorePcapFlow */
322
323 // Server latency = t(SYN Server) - t(SYN CLient)
SetServer_latency(struct FlowNode * node)324 void SetServer_latency(struct FlowNode *node) {
325 struct FlowNode *Client_node;
326 uint64_t latency;
327
328 Client_node = node->rev_node;
329 if ( !Client_node )
330 return;
331
332 latency = ((uint64_t)node->t_first.tv_sec * (uint64_t)1000000 + (uint64_t)node->t_first.tv_usec) -
333 ((uint64_t)Client_node->t_first.tv_sec * (uint64_t)1000000 + (uint64_t)Client_node->t_first.tv_usec);
334
335 node->latency.server = latency;
336 Client_node->latency.server = latency;
337 // set flag, to calc client latency with nex packet from client
338 Client_node->latency.flag = 1;
339 dbg_printf("Server latency: %llu\n", (long long unsigned)latency);
340
341 } // End of SetServerClient_latency
342
343 // Client latency = t(ACK CLient) - t(SYN Server)
SetClient_latency(struct FlowNode * node,struct timeval * t_packet)344 void SetClient_latency(struct FlowNode *node, struct timeval *t_packet) {
345 struct FlowNode *Server_node;
346 uint64_t latency;
347
348 Server_node = node->rev_node;
349 if ( !Server_node )
350 return;
351
352 latency = ((uint64_t)t_packet->tv_sec * (uint64_t)1000000 + (uint64_t)t_packet->tv_usec) -
353 ((uint64_t)Server_node->t_first.tv_sec * (uint64_t)1000000 + (uint64_t)Server_node->t_first.tv_usec);
354
355 node->latency.client = latency;
356 Server_node->latency.client = latency;
357 // reset flag
358 node->latency.flag = 0;
359 // set flag, to calc application latency with nex packet from server
360 Server_node->latency.flag = 2;
361 Server_node->latency.t_request = *t_packet;
362 dbg_printf("Client latency: %llu\n", (long long unsigned)latency);
363
364 } // End of SetClient_latency
365
366 // Application latency = t(ACK Server) - t(ACK CLient)
SetApplication_latency(struct FlowNode * node,struct timeval * t_packet)367 void SetApplication_latency(struct FlowNode *node, struct timeval *t_packet) {
368 struct FlowNode *Client_node;
369 uint64_t latency;
370
371 Client_node = node->rev_node;
372 if ( !Client_node )
373 return;
374
375 latency = ((uint64_t)t_packet->tv_sec * (uint64_t)1000000 + (uint64_t)t_packet->tv_usec) -
376 ((uint64_t)node->latency.t_request.tv_sec * (uint64_t)1000000 + (uint64_t)node->latency.t_request.tv_usec);
377
378 node->latency.application = latency;
379 Client_node->latency.application = latency;
380 // reset flag
381 node->latency.flag = 0;
382 dbg_printf("Application latency: %llu\n", (long long unsigned)latency);
383
384 } // End of SetApplication_latency
385
386
387