1 /*
2 * ndpi_util.h
3 *
4 * Copyright (C) 2011-21 - ntop.org
5 *
6 * nDPI is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU Lesser General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * nDPI is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with nDPI. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 /**
22 * This module contains routines to help setup a simple nDPI program.
23 *
24 * If you concern about performance or have to integrate nDPI in your
25 * application, you could need to reimplement them yourself.
26 *
27 * WARNING: this API is just a demo od nDPI usage: Use it at your own risk!
28 */
29 #ifndef __NDPI_UTIL_H__
30 #define __NDPI_UTIL_H__
31
32 #include "../src/lib/third_party/include/uthash.h"
33 #include <pcap.h>
34 #include "ndpi_includes.h"
35 #include "ndpi_classify.h"
36 #include "ndpi_typedefs.h"
37
38 #ifdef USE_DPDK
39 #include <rte_eal.h>
40 #include <rte_ether.h>
41 #include <rte_ethdev.h>
42 #include <rte_cycles.h>
43 #include <rte_lcore.h>
44 #include <rte_mbuf.h>
45
46 #define RX_RING_SIZE 128
47 #define TX_RING_SIZE 512
48 #define NUM_MBUFS 8191
49 #define MBUF_CACHE_SIZE 250
50 #define BURST_SIZE 32
51 #define PREFETCH_OFFSET 3
52
53 extern int dpdk_port_init(int port, struct rte_mempool *mbuf_pool);
54 extern int dpdk_port_deinit(int port);
55 #endif
56
57 #define PLEN_MAX 1504
58 #define PLEN_BIN_LEN 32
59 #define PLEN_NUM_BINS 48 /* 47*32 = 1504 */
60 #define MAX_NUM_BIN_PKTS 256
61
62 /* ETTA Spec defiintions for feature readiness */
63 #define ETTA_MIN_PACKETS 10
64 #define ETTA_MIN_OCTETS 4000
65 /** maximum line length */
66 #define LINEMAX 512
67 #define MAX_BYTE_COUNT_ARRAY_LENGTH 256
68 #define MAX_NUM_PKTS 10
69
70 #define MAX_NUM_READER_THREADS 16
71 #define IDLE_SCAN_PERIOD 10 /* msec (use TICK_RESOLUTION = 1000) */
72 #define MAX_IDLE_TIME 30000
73 #define IDLE_SCAN_BUDGET 1024
74 #define NUM_ROOTS 512
75 #define MAX_EXTRA_PACKETS_TO_CHECK 7
76 #define MAX_NDPI_FLOWS 200000000
77 #define TICK_RESOLUTION 1000
78 #define MAX_NUM_IP_ADDRESS 5 /* len of ip address array */
79 #define UPDATED_TREE 1
80 #define AGGRESSIVE_PERCENT 95.00
81 #define DIR_SRC 10
82 #define DIR_DST 20
83 #define PORT_ARRAY_SIZE 20
84 #define HOST_ARRAY_SIZE 20
85 #define FLOWS_PACKETS_THRESHOLD 0.9
86 #define FLOWS_PERCENT_THRESHOLD 1.0
87 #define FLOWS_PERCENT_THRESHOLD_2 0.2
88 #define FLOWS_THRESHOLD 1000
89 #define PKTS_PERCENT_THRESHOLD 0.1
90 #define MAX_TABLE_SIZE_1 4096
91 #define MAX_TABLE_SIZE_2 8192
92 #define INIT_VAL -1
93
94
95 // inner hash table (ja3 -> security state)
96 typedef struct ndpi_ja3_info {
97 char * ja3;
98 ndpi_cipher_weakness unsafe_cipher;
99 UT_hash_handle hh;
100 } ndpi_ja3_info;
101
102 // external hash table (host ip -> <ip string, hash table ja3c, hash table ja3s>)
103 // used to aggregate ja3 fingerprints by hosts
104 typedef struct ndpi_host_ja3_fingerprints {
105 u_int32_t ip;
106 char *ip_string;
107 char *dns_name;
108 ndpi_ja3_info *host_client_info_hasht;
109 ndpi_ja3_info *host_server_info_hasht;
110
111 UT_hash_handle hh;
112 } ndpi_host_ja3_fingerprints;
113
114
115 //inner hash table
116 typedef struct ndpi_ip_dns{
117 u_int32_t ip;
118 char *ip_string;
119 char *dns_name; //server name if any;
120 UT_hash_handle hh;
121 } ndpi_ip_dns;
122
123 //hash table ja3 -> <host, ip, security>, used to aggregate host by ja3 fingerprints
124 typedef struct ndpi_ja3_fingerprints_host{
125 char *ja3; //key
126 ndpi_cipher_weakness unsafe_cipher;
127 ndpi_ip_dns *ipToDNS_ht;
128 UT_hash_handle hh;
129 } ndpi_ja3_fingerprints_host;
130
131 struct flow_metrics {
132 float entropy, average, stddev;
133 };
134
135 struct ndpi_entropy {
136 // Entropy fields
137 pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
138 u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
139 pkt_timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
140 u_int16_t dst2src_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
141 pkt_timeval dst2src_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
142 pkt_timeval src2dst_start; /*!< first packet arrival time */
143 pkt_timeval dst2src_start; /*!< first packet arrival time */
144 u_int32_t src2dst_opackets; /*!< non-zero packet counts */
145 u_int32_t dst2src_opackets; /*!< non-zero packet counts */
146 u_int16_t src2dst_pkt_count; /*!< packet counts */
147 u_int16_t dst2src_pkt_count; /*!< packet counts */
148 u_int32_t src2dst_l4_bytes; /*!< packet counts */
149 u_int32_t dst2src_l4_bytes; /*!< packet counts */
150 u_int32_t src2dst_byte_count[MAX_BYTE_COUNT_ARRAY_LENGTH]; /*!< number of occurences of each byte */
151 u_int32_t dst2src_byte_count[MAX_BYTE_COUNT_ARRAY_LENGTH]; /*!< number of occurences of each byte */
152 u_int32_t src2dst_num_bytes;
153 u_int32_t dst2src_num_bytes;
154 double src2dst_bd_mean;
155 double src2dst_bd_variance;
156 double dst2src_bd_mean;
157 double dst2src_bd_variance;
158 float score;
159 };
160
161 // flow tracking
162 typedef struct ndpi_flow_info {
163 u_int32_t flow_id;
164 u_int32_t hashval;
165 u_int32_t src_ip;
166 u_int32_t dst_ip;
167 u_int16_t src_port;
168 u_int16_t dst_port;
169 u_int8_t detection_completed, protocol, bidirectional, check_extra_packets;
170 u_int16_t vlan_id;
171 ndpi_packet_tunnel tunnel_type;
172 struct ndpi_flow_struct *ndpi_flow;
173 char src_name[48], dst_name[48];
174 u_int8_t ip_version;
175 u_int32_t cwr_count, src2dst_cwr_count, dst2src_cwr_count;
176 u_int32_t ece_count, src2dst_ece_count, dst2src_ece_count;
177 u_int32_t urg_count, src2dst_urg_count, dst2src_urg_count;
178 u_int32_t ack_count, src2dst_ack_count, dst2src_ack_count;
179 u_int32_t psh_count, src2dst_psh_count, dst2src_psh_count;
180 u_int32_t syn_count, src2dst_syn_count, dst2src_syn_count;
181 u_int32_t fin_count, src2dst_fin_count, dst2src_fin_count;
182 u_int32_t rst_count, src2dst_rst_count, dst2src_rst_count;
183 u_int32_t c_to_s_init_win, s_to_c_init_win;
184 u_int64_t first_seen_ms, last_seen_ms;
185 u_int64_t src2dst_bytes, dst2src_bytes;
186 u_int64_t src2dst_goodput_bytes, dst2src_goodput_bytes;
187 u_int32_t src2dst_packets, dst2src_packets;
188 u_int32_t has_human_readeable_strings;
189 char human_readeable_string_buffer[32];
190
191 // result only, not used for flow identification
192 ndpi_protocol detected_protocol;
193
194 // Flow data analysis
195 struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
196 *pktlen_c_to_s, *pktlen_s_to_c;
197
198 char info[255];
199 char flow_extra_info[16];
200 char host_server_name[240];
201 char bittorent_hash[41];
202 char dhcp_fingerprint[48];
203 ndpi_risk risk;
204
205 struct {
206 u_int16_t ssl_version;
207 char client_requested_server_name[256], server_info[64],
208 client_hassh[33], server_hassh[33], *server_names,
209 *tls_alpn, *tls_supported_versions,
210 *tls_issuerDN, *tls_subjectDN,
211 ja3_client[33], ja3_server[33],
212 sha1_cert_fingerprint[20];
213 u_int8_t sha1_cert_fingerprint_set;
214 struct tls_heuristics browser_heuristics;
215
216 struct {
217 u_int16_t cipher_suite;
218 char *esni;
219 } encrypted_sni;
220
221 time_t notBefore, notAfter;
222 u_int16_t server_cipher;
223 ndpi_cipher_weakness client_unsafe_cipher, server_unsafe_cipher;
224 } ssh_tls;
225
226 struct {
227 char url[256], request_content_type[64], content_type[64], user_agent[128];
228 u_int response_status_code;
229 } http;
230
231 struct {
232 char username[32], password[32];
233 } telnet;
234
235 void *src_id, *dst_id;
236
237 struct ndpi_entropy entropy;
238 struct ndpi_entropy last_entropy;
239
240 /* Payload lenght bins */
241 #ifdef DIRECTION_BINS
242 struct ndpi_bin payload_len_bin_src2dst, payload_len_bin_dst2src;
243 #else
244 struct ndpi_bin payload_len_bin;
245 #endif
246 } ndpi_flow_info_t;
247
248
249 // flow statistics info
250 typedef struct ndpi_stats {
251 u_int32_t guessed_flow_protocols;
252 u_int64_t raw_packet_count;
253 u_int64_t ip_packet_count;
254 u_int64_t total_wire_bytes, total_ip_bytes, total_discarded_bytes;
255 u_int64_t protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
256 u_int64_t protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
257 u_int32_t protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
258 u_int32_t ndpi_flow_count;
259 u_int32_t flow_count[3];
260 u_int64_t tcp_count, udp_count;
261 u_int64_t mpls_count, pppoe_count, vlan_count, fragmented_count;
262 u_int64_t packet_len[6];
263 u_int16_t max_packet_len;
264 u_int64_t dpi_packet_count[3];
265 } ndpi_stats_t;
266
267
268 // flow preferences
269 typedef struct ndpi_workflow_prefs {
270 u_int8_t decode_tunnels;
271 u_int8_t quiet_mode;
272 u_int8_t ignore_vlanid;
273 u_int32_t num_roots;
274 u_int32_t max_ndpi_flows;
275 } ndpi_workflow_prefs_t;
276
277 struct ndpi_workflow;
278
279 /** workflow, flow, user data */
280 typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_flow_info *, void *);
281
282
283 // workflow main structure
284 typedef struct ndpi_workflow {
285 u_int64_t last_time;
286
287 struct ndpi_workflow_prefs prefs;
288 struct ndpi_stats stats;
289
290 ndpi_workflow_callback_ptr __flow_detected_callback;
291 void * __flow_detected_udata;
292 ndpi_workflow_callback_ptr __flow_giveup_callback;
293 void * __flow_giveup_udata;
294
295 /* outside referencies */
296 pcap_t *pcap_handle;
297
298 /* allocated by prefs */
299 void **ndpi_flows_root;
300 struct ndpi_detection_module_struct *ndpi_struct;
301 u_int32_t num_allocated_flows;
302 } ndpi_workflow_t;
303
304
305 /* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
306 struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle);
307
308
309 /* workflow main free function */
310 void ndpi_workflow_free(struct ndpi_workflow * workflow);
311
312
313 /** Free flow_info ndpi support structures but not the flow_info itself
314 *
315 * TODO remove! Half freeing things is bad!
316 */
317 void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
318
319
320 /* Process a packet and update the workflow */
321 struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
322 const struct pcap_pkthdr *header,
323 const u_char *packet,
324 ndpi_risk *flow_risk,
325 FILE * csv_fp);
326
327 int ndpi_is_datalink_supported(int datalink_type);
328
329 /* flow callbacks for complete detected flow
330 (ndpi_flow_info will be freed right after) */
ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow,ndpi_workflow_callback_ptr callback,void * udata)331 static inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, void * udata) {
332 workflow->__flow_detected_callback = callback;
333 workflow->__flow_detected_udata = udata;
334 }
335
336 /* flow callbacks for sufficient detected flow
337 (ndpi_flow_info will be freed right after) */
ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * workflow,ndpi_workflow_callback_ptr callback,void * udata)338 static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, void * udata) {
339 workflow->__flow_giveup_callback = callback;
340 workflow->__flow_giveup_udata = udata;
341 }
342
343 /* compare two nodes in workflow */
344 int ndpi_workflow_node_cmp(const void *a, const void *b);
345 void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow, FILE * csv_fp);
346 u_int32_t ethernet_crc32(const void* data, size_t n_bytes);
347 void ndpi_flow_info_free_data(struct ndpi_flow_info *flow);
348 void ndpi_flow_info_freer(void *node);
349 const char* print_cipher_id(u_int32_t cipher);
350 float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256], unsigned int num_bytes);
351
352 extern int nDPI_LogLevel;
353
354 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
355 #define LOG(log_level, args...) \
356 { \
357 if(log_level <= nDPI_LogLevel) \
358 printf(args); \
359 }
360 #else
361 #define LOG(...) {}
362 #endif
363
364 #endif
365