1""" 2file: ndpi.py 3This file is part of nfstream. 4 5Copyright (C) 2019-20 - nfstream.org 6 7nfstream is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 8as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. 9 10nfstream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13You should have received a copy of the GNU General Public License along with nfstream. 14If not, see <http://www.gnu.org/licenses/>. 15""" 16 17from os.path import abspath, dirname 18import cffi 19 20cc_ndpi_network_headers = """ 21struct ptr_uint32 { 22 uint32_t value; 23}; 24 25struct ndpi_chdlc 26{ 27 uint8_t addr; /* 0x0F (Unicast) - 0x8F (Broadcast) */ 28 uint8_t ctrl; /* always 0x00 */ 29 uint16_t proto_code; /* protocol type (e.g. 0x0800 IP) */ 30}; 31 32/* SLARP - Serial Line ARP http://tinyurl.com/qa54e95 */ 33struct ndpi_slarp 34{ 35 /* address requests (0x00) 36 address replies (0x01) 37 keep-alive (0x02) 38 */ 39 uint32_t slarp_type; 40 uint32_t addr_1; 41 uint32_t addr_2; 42}; 43 44/* Cisco Discovery Protocol http://tinyurl.com/qa6yw9l */ 45struct ndpi_cdp 46{ 47 uint8_t version; 48 uint8_t ttl; 49 uint16_t checksum; 50 uint16_t type; 51 uint16_t length; 52}; 53 54/* +++++++++++++++ Ethernet header (IEEE 802.3) +++++++++++++++ */ 55struct ndpi_ethhdr 56{ 57 uint8_t h_dest[6]; /* destination eth addr */ 58 uint8_t h_source[6]; /* source ether addr */ 59 uint16_t h_proto; /* data length (<= 1500) or type ID proto (>=1536) */ 60}; 61 62/* +++++++++++++++ ARP header +++++++++++++++ */ 63struct ndpi_arphdr { 64 uint16_t ar_hrd;/* Format of hardware address. */ 65 uint16_t ar_pro;/* Format of protocol address. */ 66 uint8_t ar_hln;/* Length of hardware address. */ 67 uint8_t ar_pln;/* Length of protocol address. */ 68 uint16_t ar_op;/* ARP opcode (command). */ 69 uint8_t arp_sha[6];/* sender hardware address */ 70 uint32_t arp_spa;/* sender protocol address */ 71 uint8_t arp_tha[6];/* target hardware address */ 72 uint32_t arp_tpa;/* target protocol address */ 73}; 74 75/* +++++++++++++++ DHCP header +++++++++++++++ */ 76struct ndpi_dhcphdr { 77 uint8_t msgType; 78 uint8_t htype; 79 uint8_t hlen; 80 uint8_t hops; 81 uint32_t xid;/* 4 */ 82 uint16_t secs;/* 8 */ 83 uint16_t flags; 84 uint32_t ciaddr;/* 12 */ 85 uint32_t yiaddr;/* 16 */ 86 uint32_t siaddr;/* 20 */ 87 uint32_t giaddr;/* 24 */ 88 uint8_t chaddr[16]; /* 28 */ 89 uint8_t sname[64]; /* 44 */ 90 uint8_t file[128]; /* 108 */ 91 uint32_t magic; /* 236 */ 92 uint8_t options[308]; 93}; 94 95/* +++++++++++++++ MDNS rsp header +++++++++++++++ */ 96struct ndpi_mdns_rsp_entry { 97 uint16_t rsp_type, rsp_class; 98 uint32_t ttl; 99 uint16_t data_len; 100}; 101 102/* +++++++++++++++++++ LLC header (IEEE 802.2) ++++++++++++++++ */ 103struct ndpi_snap_extension 104{ 105 uint16_t oui; 106 uint8_t oui2; 107 uint16_t proto_ID; 108}; 109 110struct ndpi_llc_header_snap 111{ 112 uint8_t dsap; 113 uint8_t ssap; 114 uint8_t ctrl; 115 struct ndpi_snap_extension snap; 116}; 117 118/* ++++++++++ RADIO TAP header (for IEEE 802.11) +++++++++++++ */ 119struct ndpi_radiotap_header 120{ 121 uint8_t version; /* set to 0 */ 122 uint8_t pad; 123 uint16_t len; 124 uint32_t present; 125 uint64_t MAC_timestamp; 126 uint8_t flags; 127}; 128 129/* ++++++++++++ Wireless header (IEEE 802.11) ++++++++++++++++ */ 130struct ndpi_wifi_header 131{ 132 uint16_t fc; 133 uint16_t duration; 134 uint8_t rcvr[6]; 135 uint8_t trsm[6]; 136 uint8_t dest[6]; 137 uint16_t seq_ctrl; 138 /* uint64_t ccmp - for data encryption only - check fc.flag */ 139}; 140 141/* +++++++++++++++++++++++ MPLS header +++++++++++++++++++++++ */ 142struct ndpi_mpls_header 143{ 144 /* Before using this strcut to parse an MPLS header, you will need to convert 145 * the 4-byte data to the correct endianess with ntohl(). */ 146 uint32_t ttl:8, s:1, exp:3, label:20; 147}; 148 149extern union mpls { 150 uint32_t u32; 151 struct ndpi_mpls_header mpls; 152} mpls; 153 154/* ++++++++++++++++++++++++ IP header ++++++++++++++++++++++++ */ 155struct ndpi_iphdr { 156 uint8_t ihl:4, version:4; 157 uint8_t tos; 158 uint16_t tot_len; 159 uint16_t id; 160 uint16_t frag_off; 161 uint8_t ttl; 162 uint8_t protocol; 163 uint16_t check; 164 uint32_t saddr; 165 uint32_t daddr; 166}; 167 168/* +++++++++++++++++++++++ IPv6 header +++++++++++++++++++++++ */ 169/* rfc3542 */ 170struct ndpi_in6_addr { 171 union { 172 uint8_t u6_addr8[16]; 173 uint16_t u6_addr16[8]; 174 uint32_t u6_addr32[4]; 175 uint64_t u6_addr64[2]; 176 } u6_addr; /* 128-bit IP6 address */ 177}; 178 179struct ndpi_ip6_hdrctl { 180 uint32_t ip6_un1_flow; 181 uint16_t ip6_un1_plen; 182 uint8_t ip6_un1_nxt; 183 uint8_t ip6_un1_hlim; 184}; 185 186struct ndpi_ipv6hdr { 187 struct ndpi_ip6_hdrctl ip6_hdr; 188 struct ndpi_in6_addr ip6_src; 189 struct ndpi_in6_addr ip6_dst; 190}; 191 192/* +++++++++++++++++++++++ TCP header +++++++++++++++++++++++ */ 193struct ndpi_tcphdr 194{ 195 uint16_t source; 196 uint16_t dest; 197 uint32_t seq; 198 uint32_t ack_seq; 199 uint16_t res1:4, doff:4, fin:1, syn:1, rst:1, psh:1, ack:1, urg:1, ece:1, cwr:1; 200 uint16_t window; 201 uint16_t check; 202 uint16_t urg_ptr; 203}; 204 205/* +++++++++++++++++++++++ UDP header +++++++++++++++++++++++ */ 206struct ndpi_udphdr 207{ 208 uint16_t source; 209 uint16_t dest; 210 uint16_t len; 211 uint16_t check; 212}; 213struct ndpi_dns_packet_header { 214 uint16_t tr_id; 215 uint16_t flags; 216 uint16_t num_queries; 217 uint16_t num_answers; 218 uint16_t authority_rrs; 219 uint16_t additional_rrs; 220}; 221 222/* +++++++++++++++++++++++ ICMP header +++++++++++++++++++++++ */ 223struct ndpi_icmphdr { 224 uint8_t type;/* message type */ 225 uint8_t code;/* type sub-code */ 226 uint16_t checksum; 227 union { 228 struct { 229 uint16_t id; 230 uint16_t sequence; 231 } echo; /* echo datagram */ 232 233 uint32_t gateway; /* gateway address */ 234 struct { 235 uint16_t _unused; 236 uint16_t mtu; 237 } frag;/* path mtu discovery */ 238 } un; 239}; 240 241/* +++++++++++++++++++++++ ICMP6 header +++++++++++++++++++++++ */ 242struct ndpi_icmp6hdr { 243 uint8_t icmp6_type; /* type field */ 244 uint8_t icmp6_code; /* code field */ 245 uint16_t icmp6_cksum; /* checksum field */ 246 union { 247 uint32_t icmp6_un_data32[1]; /* type-specific field */ 248 uint16_t icmp6_un_data16[2]; /* type-specific field */ 249 uint8_t icmp6_un_data8[4]; /* type-specific field */ 250 } icmp6_dataun; 251}; 252 253/* +++++++++++++++++++++++ VXLAN header +++++++++++++++++++++++ */ 254struct ndpi_vxlanhdr { 255 uint16_t flags; 256 uint16_t groupPolicy; 257 uint32_t vni; 258}; 259 260struct tinc_cache_entry { 261 uint32_t src_address; 262 uint32_t dst_address; 263 uint16_t dst_port; 264}; 265""" 266 267cc_ndpi_stuctures = """ 268 269#define NDPI_MAX_NUM_TLS_APPL_BLOCKS 8 270 271typedef enum { 272 NDPI_LOG_ERROR, 273 NDPI_LOG_TRACE, 274 NDPI_LOG_DEBUG, 275 NDPI_LOG_DEBUG_EXTRA 276} ndpi_log_level_t; 277 278typedef enum { 279 ndpi_l4_proto_unknown = 0, 280 ndpi_l4_proto_tcp_only, 281 ndpi_l4_proto_udp_only, 282 ndpi_l4_proto_tcp_and_udp, 283} ndpi_l4_proto_info; 284 285typedef enum { 286 ndpi_no_tunnel = 0, 287 ndpi_gtp_tunnel, 288 ndpi_capwap_tunnel, 289 ndpi_tzsp_tunnel, 290 ndpi_l2tp_tunnel, 291} ndpi_packet_tunnel; 292 293typedef enum { 294 NDPI_NO_RISK = 0, 295 NDPI_URL_POSSIBLE_XSS, 296 NDPI_URL_POSSIBLE_SQL_INJECTION, 297 NDPI_URL_POSSIBLE_RCE_INJECTION, 298 NDPI_BINARY_APPLICATION_TRANSFER, 299 NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT, 300 NDPI_TLS_SELFSIGNED_CERTIFICATE, 301 NDPI_TLS_OBSOLETE_VERSION, 302 NDPI_TLS_WEAK_CIPHER, 303 NDPI_TLS_CERTIFICATE_EXPIRED, 304 NDPI_TLS_CERTIFICATE_MISMATCH, 305 NDPI_HTTP_SUSPICIOUS_USER_AGENT, 306 NDPI_HTTP_NUMERIC_IP_HOST, 307 NDPI_HTTP_SUSPICIOUS_URL, 308 NDPI_HTTP_SUSPICIOUS_HEADER, 309 NDPI_TLS_NOT_CARRYING_HTTPS, 310 NDPI_SUSPICIOUS_DGA_DOMAIN, 311 NDPI_MALFORMED_PACKET, 312 NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER, 313 NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, 314 NDPI_SMB_INSECURE_VERSION, 315 NDPI_TLS_SUSPICIOUS_ESNI_USAGE, 316 NDPI_UNSAFE_PROTOCOL, 317 NDPI_DNS_SUSPICIOUS_TRAFFIC, 318 NDPI_TLS_MISSING_SNI, 319 NDPI_HTTP_SUSPICIOUS_CONTENT, 320 NDPI_RISKY_ASN, 321 NDPI_RISKY_DOMAIN, 322 NDPI_MALICIOUS_JA3, 323 NDPI_MALICIOUS_SHA1_CERTIFICATE, 324 NDPI_DESKTOP_OR_FILE_SHARING_SESSION, 325 326 /* Leave this as last member */ 327 NDPI_MAX_RISK 328} ndpi_risk_enum; 329 330typedef uint32_t ndpi_risk; 331 332/* NDPI_VISIT */ 333typedef enum { 334 ndpi_preorder, 335 ndpi_postorder, 336 ndpi_endorder, 337 ndpi_leaf 338} ndpi_VISIT; 339 340/* NDPI_NODE */ 341typedef struct node_t { 342 char *key; 343 struct node_t *left, *right; 344} ndpi_node; 345 346/* NDPI_MASK_SIZE */ 347typedef uint32_t ndpi_ndpi_mask; 348 349/* NDPI_PROTO_BITMASK_STRUCT */ 350typedef struct ndpi_protocol_bitmask_struct { 351 ndpi_ndpi_mask fds_bits[16]; 352} NDPI_PROTOCOL_BITMASK; 353 354/* NDPI_PROTOCOL_BITTORRENT */ 355typedef struct spinlock { 356 volatile int val; 357} spinlock_t; 358 359typedef struct atomic { 360 volatile int counter; 361} atomic_t; 362 363typedef long int time_t; 364 365struct hash_ip4p_node { 366 struct hash_ip4p_node *next, *prev; 367 time_t lchg; 368 uint16_t port,count:12,flag:4; 369 uint32_t ip; 370}; 371 372struct hash_ip4p { 373 struct hash_ip4p_node *top; 374 spinlock_t lock; 375 size_t len; 376}; 377 378struct hash_ip4p_table { 379 size_t size; 380 int ipv6; 381 spinlock_t lock; 382 atomic_t count; 383 struct hash_ip4p tbl; 384}; 385 386struct bt_announce { // 192 bytes 387 uint32_t hash[5]; 388 uint32_t ip[4]; 389 uint32_t time; 390 uint16_t port; 391 uint8_t name_len, 392 name[149]; // 149 bytes 393}; 394 395/* NDPI_PROTOCOL_TINC */ 396#define TINC_CACHE_MAX_SIZE 10 397 398typedef enum { 399 NDPI_HTTP_METHOD_UNKNOWN = 0, 400 NDPI_HTTP_METHOD_OPTIONS, 401 NDPI_HTTP_METHOD_GET, 402 NDPI_HTTP_METHOD_HEAD, 403 NDPI_HTTP_METHOD_PATCH, 404 NDPI_HTTP_METHOD_POST, 405 NDPI_HTTP_METHOD_PUT, 406 NDPI_HTTP_METHOD_DELETE, 407 NDPI_HTTP_METHOD_TRACE, 408 NDPI_HTTP_METHOD_CONNECT 409} ndpi_http_method; 410 411struct ndpi_lru_cache_entry { 412 uint32_t key; /* Store the whole key to avoid ambiguities */ 413 uint32_t is_full:1, value:16, pad:15; 414}; 415 416struct ndpi_lru_cache { 417 uint32_t num_entries; 418 struct ndpi_lru_cache_entry *entries; 419}; 420 421typedef union 422{ 423 uint32_t ipv4; 424 uint8_t ipv4_uint8_t[4]; 425 struct ndpi_in6_addr ipv6; 426} ndpi_ip_addr_t; 427 428struct ndpi_id_struct { 429 /** 430 detected_protocol_bitmask: 431 access this bitmask to find out whether an id has used skype or not 432 if a flag is set here, it will not be reset 433 to compare this, use: 434 **/ 435 NDPI_PROTOCOL_BITMASK detected_protocol_bitmask; 436 /* NDPI_PROTOCOL_RTSP */ 437 ndpi_ip_addr_t rtsp_ip_address; 438 439 /* NDPI_PROTOCOL_YAHOO */ 440 uint32_t yahoo_video_lan_timer; 441 442 /* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */ 443 /* NDPI_PROTOCOL_IRC */ 444#define NDPI_PROTOCOL_IRC_MAXPORT 8 445 uint16_t irc_port[NDPI_PROTOCOL_IRC_MAXPORT]; 446 uint32_t last_time_port_used[NDPI_PROTOCOL_IRC_MAXPORT]; 447 uint32_t irc_ts; 448 449 /* NDPI_PROTOCOL_GNUTELLA */ 450 uint32_t gnutella_ts; 451 452 /* NDPI_PROTOCOL_THUNDER */ 453 uint32_t thunder_ts; 454 455 /* NDPI_PROTOCOL_RTSP */ 456 uint32_t rtsp_timer; 457 458 /* NDPI_PROTOCOL_ZATTOO */ 459 uint32_t zattoo_ts; 460 461 /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ 462 uint32_t jabber_stun_or_ft_ts; 463 464 /* NDPI_PROTOCOL_DIRECTCONNECT */ 465 uint32_t directconnect_last_safe_access_time; 466 467 /* NDPI_PROTOCOL_SOULSEEK */ 468 uint32_t soulseek_last_safe_access_time; 469 470 /* NDPI_PROTOCOL_DIRECTCONNECT */ 471 uint16_t detected_directconnect_port; 472 uint16_t detected_directconnect_udp_port; 473 uint16_t detected_directconnect_ssl_port; 474 475 /* NDPI_PROTOCOL_BITTORRENT */ 476#define NDPI_BT_PORTS 8 477 uint16_t bt_port_t[NDPI_BT_PORTS]; 478 uint16_t bt_port_u[NDPI_BT_PORTS]; 479 480 /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ 481#define JABBER_MAX_STUN_PORTS 6 482 uint16_t jabber_voice_stun_port[JABBER_MAX_STUN_PORTS]; 483 uint16_t jabber_file_transfer_port[2]; 484 485 /* NDPI_PROTOCOL_GNUTELLA */ 486 uint16_t detected_gnutella_port; 487 488 /* NDPI_PROTOCOL_GNUTELLA */ 489 uint16_t detected_gnutella_udp_port1; 490 uint16_t detected_gnutella_udp_port2; 491 492 /* NDPI_PROTOCOL_SOULSEEK */ 493 uint16_t soulseek_listen_port; 494 495 /* NDPI_PROTOCOL_IRC */ 496 uint8_t irc_number_of_port; 497 498 /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ 499 uint8_t jabber_voice_stun_used_ports; 500 501 /* NDPI_PROTOCOL_SIP */ 502 /* NDPI_PROTOCOL_YAHOO */ 503 uint32_t yahoo_video_lan_dir:1; 504 505 /* NDPI_PROTOCOL_YAHOO */ 506 uint32_t yahoo_conf_logged_in:1; 507 uint32_t yahoo_voice_conf_logged_in:1; 508 509 /* NDPI_PROTOCOL_RTSP */ 510 uint32_t rtsp_ts_set:1; 511}; 512 513struct ndpi_flow_tcp_struct { 514 /* NDPI_PROTOCOL_MAIL_SMTP */ 515 uint16_t smtp_command_bitmask; 516 517 /* NDPI_PROTOCOL_MAIL_POP */ 518 uint16_t pop_command_bitmask; 519 520 /* NDPI_PROTOCOL_QQ */ 521 uint16_t qq_nxt_len; 522 523 /* NDPI_PROTOCOL_WHATSAPP */ 524 uint8_t wa_matched_so_far; 525 526 /* NDPI_PROTOCOL_TDS */ 527 uint8_t tds_login_version; 528 529 /* NDPI_PROTOCOL_IRC */ 530 uint8_t irc_stage; 531 uint8_t irc_port; 532 533 /* NDPI_PROTOCOL_H323 */ 534 uint8_t h323_valid_packets; 535 536 /* NDPI_PROTOCOL_GNUTELLA */ 537 uint8_t gnutella_msg_id[3]; 538 539 /* NDPI_PROTOCOL_IRC */ 540 uint32_t irc_3a_counter:3; 541 uint32_t irc_stage2:5; 542 uint32_t irc_direction:2; 543 uint32_t irc_0x1000_full:1; 544 545 /* NDPI_PROTOCOL_SOULSEEK */ 546 uint32_t soulseek_stage:2; 547 548 /* NDPI_PROTOCOL_TDS */ 549 uint32_t tds_stage:3; 550 551 /* NDPI_PROTOCOL_USENET */ 552 uint32_t usenet_stage:2; 553 554 /* NDPI_PROTOCOL_IMESH */ 555 uint32_t imesh_stage:4; 556 557 /* NDPI_PROTOCOL_HTTP */ 558 uint32_t http_setup_dir:2; 559 uint32_t http_stage:2; 560 uint32_t http_empty_line_seen:1; 561 uint32_t http_wait_for_retransmission:1; 562 563 /* NDPI_PROTOCOL_GNUTELLA */ 564 uint32_t gnutella_stage:2; // 0 - 2 565 566 /* NDPI_CONTENT_MMS */ 567 uint32_t mms_stage:2; 568 569 /* NDPI_PROTOCOL_YAHOO */ 570 uint32_t yahoo_sip_comm:1; 571 uint32_t yahoo_http_proxy_stage:2; 572 573 /* NDPI_PROTOCOL_MSN */ 574 uint32_t msn_stage:3; 575 uint32_t msn_ssl_ft:2; 576 577 /* NDPI_PROTOCOL_SSH */ 578 uint32_t ssh_stage:3; 579 580 /* NDPI_PROTOCOL_VNC */ 581 uint32_t vnc_stage:2; // 0 - 3 582 583 /* NDPI_PROTOCOL_TELNET */ 584 uint32_t telnet_stage:2; // 0 - 2 585 586 struct { 587 struct { 588 uint8_t *buffer; 589 unsigned buffer_len, buffer_used; 590 } message; 591 592 void* srv_cert_fingerprint_ctx; /* SHA-1 */ 593 594 /* NDPI_PROTOCOL_TLS */ 595 uint8_t hello_processed:1, certificate_processed:1, subprotocol_detected:1, fingerprint_set:1, _pad:4; 596 uint8_t sha1_certificate_fingerprint[20], num_tls_blocks; 597 int16_t tls_application_blocks_len[NDPI_MAX_NUM_TLS_APPL_BLOCKS]; 598 } tls; 599 600 /* NDPI_PROTOCOL_POSTGRES */ 601 uint32_t postgres_stage:3; 602 603 /* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */ 604 uint32_t ddlink_server_direction:1; 605 uint32_t seen_syn:1; 606 uint32_t seen_syn_ack:1; 607 uint32_t seen_ack:1; 608 609 /* NDPI_PROTOCOL_ICECAST */ 610 uint32_t icecast_stage:1; 611 612 /* NDPI_PROTOCOL_DOFUS */ 613 uint32_t dofus_stage:1; 614 615 /* NDPI_PROTOCOL_FIESTA */ 616 uint32_t fiesta_stage:2; 617 618 /* NDPI_PROTOCOL_WORLDOFWARCRAFT */ 619 uint32_t wow_stage:2; 620 621 /* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */ 622 uint32_t veoh_tv_stage:2; 623 624 /* NDPI_PROTOCOL_SHOUTCAST */ 625 uint32_t shoutcast_stage:2; 626 627 /* NDPI_PROTOCOL_RTP */ 628 uint32_t rtp_special_packets_seen:1; 629 630 /* NDPI_PROTOCOL_MAIL_POP */ 631 uint32_t mail_pop_stage:2; 632 633 /* NDPI_PROTOCOL_MAIL_IMAP */ 634 uint32_t mail_imap_stage:3, mail_imap_starttls:2; 635 636 /* NDPI_PROTOCOL_SKYPE */ 637 uint8_t skype_packet_id; 638 639 /* NDPI_PROTOCOL_CITRIX */ 640 uint8_t citrix_packet_id; 641 642 /* NDPI_PROTOCOL_LOTUS_NOTES */ 643 uint8_t lotus_notes_packet_id; 644 645 /* NDPI_PROTOCOL_TEAMVIEWER */ 646 uint8_t teamviewer_stage; 647 648 /* NDPI_PROTOCOL_ZMQ */ 649 uint8_t prev_zmq_pkt_len; 650 uint8_t prev_zmq_pkt[10]; 651 652 /* NDPI_PROTOCOL_PPSTREAM */ 653 uint32_t ppstream_stage:3; 654 655 /* NDPI_PROTOCOL_MEMCACHED */ 656 uint8_t memcached_matches; 657 658 /* NDPI_PROTOCOL_NEST_LOG_SINK */ 659 uint8_t nest_log_sink_matches; 660}; 661 662struct ndpi_flow_udp_struct { 663 /* NDPI_PROTOCOL_SNMP */ 664 uint32_t snmp_msg_id; 665 666 /* NDPI_PROTOCOL_SNMP */ 667 uint32_t snmp_stage:2; 668 669 /* NDPI_PROTOCOL_PPSTREAM */ 670 uint32_t ppstream_stage:3; // 0 - 7 671 672 /* NDPI_PROTOCOL_HALFLIFE2 */ 673 uint32_t halflife2_stage:2; // 0 - 2 674 675 /* NDPI_PROTOCOL_TFTP */ 676 uint32_t tftp_stage:2; 677 678 /* NDPI_PROTOCOL_AIMINI */ 679 uint32_t aimini_stage:5; 680 681 /* NDPI_PROTOCOL_XBOX */ 682 uint32_t xbox_stage:1; 683 684 /* NDPI_PROTOCOL_WINDOWS_UPDATE */ 685 uint32_t wsus_stage:1; 686 687 /* NDPI_PROTOCOL_SKYPE */ 688 uint8_t skype_packet_id; 689 690 /* NDPI_PROTOCOL_TEAMVIEWER */ 691 uint8_t teamviewer_stage; 692 693 /* NDPI_PROTOCOL_EAQ */ 694 uint8_t eaq_pkt_id; 695 uint32_t eaq_sequence; 696 697 /* NDPI_PROTOCOL_RX */ 698 uint32_t rx_conn_epoch; 699 uint32_t rx_conn_id; 700 701 /* NDPI_PROTOCOL_MEMCACHED */ 702 uint8_t memcached_matches; 703 704 /* NDPI_PROTOCOL_WIREGUARD */ 705 uint8_t wireguard_stage; 706 uint32_t wireguard_peer_index[2]; 707}; 708 709struct ndpi_int_one_line_struct { 710 const uint8_t *ptr; 711 uint16_t len; 712}; 713 714struct ndpi_packet_struct { 715 const struct ndpi_iphdr *iph; 716 const struct ndpi_ipv6hdr *iphv6; 717 const struct ndpi_tcphdr *tcp; 718 const struct ndpi_udphdr *udp; 719 const uint8_t *generic_l4_ptr; /* is set only for non tcp-udp traffic */ 720 const uint8_t *payload; 721 722 uint64_t current_time_ms; 723 724 uint16_t detected_protocol_stack[2]; 725 uint16_t protocol_stack_info; 726 727 struct ndpi_int_one_line_struct line[64]; 728 /* HTTP headers */ 729 struct ndpi_int_one_line_struct host_line; 730 struct ndpi_int_one_line_struct forwarded_line; 731 struct ndpi_int_one_line_struct referer_line; 732 struct ndpi_int_one_line_struct content_line; 733 struct ndpi_int_one_line_struct content_disposition_line; 734 struct ndpi_int_one_line_struct accept_line; 735 struct ndpi_int_one_line_struct user_agent_line; 736 struct ndpi_int_one_line_struct http_url_name; 737 struct ndpi_int_one_line_struct http_encoding; 738 struct ndpi_int_one_line_struct http_transfer_encoding; 739 struct ndpi_int_one_line_struct http_contentlen; 740 struct ndpi_int_one_line_struct http_cookie; 741 struct ndpi_int_one_line_struct http_origin; 742 struct ndpi_int_one_line_struct http_x_session_type; 743 struct ndpi_int_one_line_struct server_line; 744 struct ndpi_int_one_line_struct http_method; 745 struct ndpi_int_one_line_struct http_response; 746 uint8_t http_num_headers; /* number of found (valid) header lines in HTTP request or response */ 747 748 uint16_t l3_packet_len; 749 uint16_t l4_packet_len; 750 uint16_t payload_packet_len; 751 uint16_t actual_payload_len; 752 uint16_t num_retried_bytes; 753 uint16_t parsed_lines; 754 uint16_t parsed_unix_lines; 755 uint16_t empty_line_position; 756 uint8_t tcp_retransmission; 757 uint8_t l4_protocol; 758 759 uint8_t tls_certificate_detected:4, tls_certificate_num_checks:4; 760 uint8_t packet_lines_parsed_complete:1, 761 packet_direction:1, empty_line_position_set:1, pad:5; 762}; 763 764struct ndpi_detection_module_struct; 765struct ndpi_flow_struct; 766 767struct ndpi_call_function_struct { 768 NDPI_PROTOCOL_BITMASK detection_bitmask; 769 NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask; 770 uint32_t ndpi_selection_bitmask; 771 void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); 772 uint8_t detection_feature; 773}; 774 775struct ndpi_subprotocol_conf_struct { 776 void (*func) (struct ndpi_detection_module_struct *, char *attr, char *value, int protocol_id); 777}; 778 779typedef struct { 780 uint16_t port_low, port_high; 781} ndpi_port_range; 782 783typedef enum { 784 NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ 785 NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ 786 NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy */ 787 NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols 788 with clear pass should be here (e.g., telnet) */ 789 NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Possibly dangerous (ex. Tor). */ 790 NDPI_PROTOCOL_DANGEROUS, /* Surely is dangerous (ex. smbv1). Be prepared to troubles */ 791 NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ 792 NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ 793} ndpi_protocol_breed_t; 794 795#define NUM_BREEDS 8 796 797/* Abstract categories to group the protocols. */ 798typedef enum { 799 NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ 800 NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ 801 NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ 802 NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ 803 NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ 804 NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ 805 NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ 806 NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ 807 NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ 808 NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ 809 NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ 810 NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ 811 NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ 812 NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ 813 NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ 814 NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ 815 NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ 816 NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ 817 NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ 818 NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ 819 820 /* See #define NUM_CUSTOM_CATEGORIES */ 821 NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ 822 NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ 823 NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ 824 NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ 825 NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ 826 827 /* Further categories... */ 828 NDPI_PROTOCOL_CATEGORY_MUSIC, 829 NDPI_PROTOCOL_CATEGORY_VIDEO, 830 NDPI_PROTOCOL_CATEGORY_SHOPPING, 831 NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY, 832 NDPI_PROTOCOL_CATEGORY_FILE_SHARING, 833 834 /* 835 The category below is used by sites who are used 836 to test connectivity 837 */ 838 NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, 839 840 /* Some custom categories */ 841 CUSTOM_CATEGORY_MINING = 99, 842 CUSTOM_CATEGORY_MALWARE = 100, 843 CUSTOM_CATEGORY_ADVERTISEMENT = 101, 844 CUSTOM_CATEGORY_BANNED_SITE = 102, 845 CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, 846 CUSTOM_CATEGORY_ALLOWED_SITE = 104, 847 /* 848 The category below is used to track communications made by 849 security applications (e.g. sophosxl.net, spamhaus.org) 850 to track malware, spam etc. 851 */ 852 CUSTOM_CATEGORY_ANTIMALWARE = 105, 853 854 /* 855 IMPORTANT 856 Please keep in sync with 857 static const char* categories[] = { ..} 858 in ndpi_main.c 859 */ 860 861 NDPI_PROTOCOL_NUM_CATEGORIES 862 /* 863 NOTE: Keep this as last member 864 Unused as value but useful to getting the number of elements 865 in this datastructure 866 */ 867} ndpi_protocol_category_t; 868 869typedef enum { 870 ndpi_pref_direction_detect_disable = 0, 871 ndpi_pref_enable_tls_block_dissection 872} ndpi_detection_preference; 873 874/* ntop extensions */ 875typedef struct ndpi_proto_defaults { 876 char *protoName; 877 ndpi_protocol_category_t protoCategory; 878 u_int16_t * subprotocols; 879 size_t subprotocol_count; 880 uint16_t protoId, protoIdx; 881 uint16_t tcp_default_ports[5], udp_default_ports[5]; 882 ndpi_protocol_breed_t protoBreed; 883 void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); 884} ndpi_proto_defaults_t; 885 886typedef struct ndpi_default_ports_tree_node { 887 ndpi_proto_defaults_t *proto; 888 uint8_t customUserProto; 889 uint16_t default_port; 890} ndpi_default_ports_tree_node_t; 891 892typedef struct _ndpi_automa { 893 void *ac_automa; /* Real type is AC_AUTOMATA_t */ 894 uint8_t ac_automa_finalized; 895} ndpi_automa; 896 897typedef struct ndpi_proto { 898 /* 899 Note 900 below we do not use ndpi_protocol_id_t as users can define their own 901 custom protocols and thus the typedef could be too short in size. 902 */ 903 uint16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */; 904 ndpi_protocol_category_t category; 905} ndpi_protocol; 906 907#define NUM_CUSTOM_CATEGORIES 5 908#define CUSTOM_CATEGORY_LABEL_LEN 32 909 910 911struct ndpi_detection_module_struct { 912 NDPI_PROTOCOL_BITMASK detection_bitmask; 913 NDPI_PROTOCOL_BITMASK generic_http_packet_bitmask; 914 915 uint32_t current_ts; 916 917 uint32_t ticks_per_second; 918 919 uint16_t num_tls_blocks_to_follow; 920 921 char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN]; 922 /* callback function buffer */ 923 struct ndpi_call_function_struct callback_buffer[250]; 924 uint32_t callback_buffer_size; 925 926 struct ndpi_call_function_struct callback_buffer_tcp_no_payload[250]; 927 uint32_t callback_buffer_size_tcp_no_payload; 928 929 struct ndpi_call_function_struct callback_buffer_tcp_payload[250]; 930 uint32_t callback_buffer_size_tcp_payload; 931 932 struct ndpi_call_function_struct callback_buffer_udp[250]; 933 uint32_t callback_buffer_size_udp; 934 935 struct ndpi_call_function_struct callback_buffer_non_tcp_udp[250]; 936 uint32_t callback_buffer_size_non_tcp_udp; 937 938 ndpi_default_ports_tree_node_t *tcpRoot, *udpRoot; 939 940 ndpi_log_level_t ndpi_log_level; /* default error */ 941 942 /* misc parameters */ 943 uint32_t tcp_max_retransmission_window_size; 944 945 uint32_t directconnect_connection_ip_tick_timeout; 946 947 /* subprotocol registration handler */ 948 struct ndpi_subprotocol_conf_struct subprotocol_conf[250]; 949 950 unsigned ndpi_num_supported_protocols; 951 unsigned ndpi_num_custom_protocols; 952 953 /* HTTP/DNS/HTTPS host matching */ 954 ndpi_automa host_automa, /* Used for DNS/HTTPS */ 955 content_automa, /* Used for HTTP subprotocol_detection */ 956 subprotocol_automa, /* Used for HTTP subprotocol_detection */ 957 bigrams_automa, impossible_bigrams_automa; /* TOR */ 958 /* IMPORTANT: please update ndpi_finalize_initalization() whenever you add a new automa */ 959 960 struct { 961 ndpi_automa hostnames, hostnames_shadow; 962 void *ipAddresses, *ipAddresses_shadow; /* Patricia */ 963 uint8_t categories_loaded; 964 } custom_categories; 965 966 /* IP-based protocol detection */ 967 void *protocols_ptree; 968 969 /* irc parameters */ 970 uint32_t irc_timeout; 971 /* gnutella parameters */ 972 uint32_t gnutella_timeout; 973 /* thunder parameters */ 974 uint32_t thunder_timeout; 975 /* SoulSeek parameters */ 976 uint32_t soulseek_connection_ip_tick_timeout; 977 /* rtsp parameters */ 978 uint32_t rtsp_connection_timeout; 979 /* rstp */ 980 uint32_t orb_rstp_ts_timeout; 981 /* yahoo */ 982 uint8_t yahoo_detect_http_connections; 983 uint32_t yahoo_lan_video_timeout; 984 uint32_t zattoo_connection_timeout; 985 uint32_t jabber_stun_timeout; 986 uint32_t jabber_file_transfer_timeout; 987 uint8_t ip_version_limit; 988 /* NDPI_PROTOCOL_BITTORRENT */ 989 struct hash_ip4p_table *bt_ht; 990 struct hash_ip4p_table *bt6_ht; 991 /* BT_ANNOUNCE */ 992 struct bt_announce *bt_ann; 993 int bt_ann_len; 994 995 /* NDPI_PROTOCOL_OOKLA */ 996 struct ndpi_lru_cache *ookla_cache; 997 998 /* NDPI_PROTOCOL_TINC */ 999 struct cache *tinc_cache; 1000 1001 /* NDPI_PROTOCOL_STUN and subprotocols */ 1002 struct ndpi_lru_cache *stun_cache; 1003 1004 /* NDPI_PROTOCOL_MSTEAMS */ 1005 struct ndpi_lru_cache *msteams_cache; 1006 1007 ndpi_proto_defaults_t proto_defaults[512]; 1008 1009 uint8_t direction_detect_disable:1, /* disable internal detection of packet direction */ 1010 _pad:7; 1011}; 1012 1013#define NDPI_CIPHER_SAFE 0 1014#define NDPI_CIPHER_WEAK 1 1015#define NDPI_CIPHER_INSECURE 2 1016 1017typedef enum { 1018 ndpi_cipher_safe = NDPI_CIPHER_SAFE, 1019 ndpi_cipher_weak = NDPI_CIPHER_WEAK, 1020 ndpi_cipher_insecure = NDPI_CIPHER_INSECURE 1021} ndpi_cipher_weakness; 1022 1023struct ndpi_flow_struct { 1024 uint16_t detected_protocol_stack[2]; 1025 uint16_t protocol_stack_info; 1026 /* init parameter, internal used to set up timestamp,... */ 1027 uint16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category; 1028 uint8_t l4_proto, protocol_id_already_guessed:1, host_already_guessed:1, 1029 init_finished:1, setup_packet_direction:1, packet_direction:1, check_extra_packets:1; 1030 /* 1031 if ndpi_struct->direction_detect_disable == 1 1032 tcp sequence number connection tracking 1033 */ 1034 uint32_t next_tcp_seq_nr[2]; 1035 uint8_t max_extra_packets_to_check; 1036 uint8_t num_extra_packets_checked; 1037 uint8_t num_processed_pkts; /* <= WARNING it can wrap but we do expect people to giveup earlier */ 1038 1039 int (*extra_packets_func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); 1040 /* 1041 the tcp / udp / other l4 value union 1042 used to reduce the number of bytes for tcp or udp protocol states 1043 */ 1044 union { 1045 struct ndpi_flow_tcp_struct tcp; 1046 struct ndpi_flow_udp_struct udp; 1047 } l4; 1048 1049 /* Place textual flow info here */ 1050 char flow_extra_info[16]; 1051 1052 /* 1053 Pointer to src or dst that identifies the 1054 server of this connection 1055 */ 1056 struct ndpi_id_struct *server_id; 1057 /* HTTP host or DNS query */ 1058 uint8_t host_server_name[240]; 1059 uint8_t initial_binary_bytes[8], initial_binary_bytes_len; 1060 uint8_t risk_checked; 1061 ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */ 1062 1063 /* 1064 This structure below will not stay inside the protos 1065 structure below as HTTP is used by many subprotocols 1066 such as Facebook, Google... so it is hard to know 1067 when to use it or not. Thus we leave it outside for the 1068 time being. 1069 */ 1070 struct { 1071 ndpi_http_method method; 1072 char *url, *content_type, *user_agent; 1073 uint8_t num_request_headers, num_response_headers; 1074 uint8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */ 1075 uint16_t response_status_code; /* 200, 404, etc. */ 1076 uint8_t detected_os[32]; /* Via HTTP/QUIC User-Agent */ 1077 1078 } http; 1079 1080 /* 1081 Put outside of the union to avoid issues in case the protocol 1082 is remapped to somethign pther than Kerberos due to a faulty 1083 dissector 1084 */ 1085 struct { 1086 char *pktbuf; 1087 uint16_t pktbuf_maxlen, pktbuf_currlen; 1088 } kerberos_buf; 1089 union { 1090 /* the only fields useful for nDPI and ntopng */ 1091 struct { 1092 uint8_t num_queries, num_answers, reply_code, is_query; 1093 uint16_t query_type, query_class, rsp_type; 1094 ndpi_ip_addr_t rsp_addr; /* The first address in a DNS response packet */ 1095 } dns; 1096 1097 struct { 1098 uint8_t request_code; 1099 uint8_t version; 1100 } ntp; 1101 1102 struct { 1103 char hostname[48], domain[48], username[48]; 1104 } kerberos; 1105 1106 struct { 1107 struct { 1108 char ssl_version_str[12]; 1109 uint16_t ssl_version, server_names_len; 1110 char client_requested_server_name[64], *server_names, 1111 *alpn, *tls_supported_versions, *issuerDN, *subjectDN; 1112 uint32_t notBefore, notAfter; 1113 char ja3_client[33], ja3_server[33]; 1114 uint16_t server_cipher; 1115 struct { 1116 uint16_t cipher_suite; 1117 char *esni; 1118 } encrypted_sni; 1119 ndpi_cipher_weakness server_unsafe_cipher; 1120 } ssl; 1121 1122 struct { 1123 uint8_t num_udp_pkts, num_processed_pkts, num_binding_requests; 1124 } stun; 1125 1126 /* We can have STUN over SSL/TLS thus they need to live together */ 1127 } stun_ssl; 1128 1129 struct { 1130 char client_signature[48], server_signature[48]; 1131 char hassh_client[33], hassh_server[33]; 1132 } ssh; 1133 1134 struct { 1135 uint8_t last_one_byte_pkt, last_byte; 1136 } imo; 1137 1138 struct { 1139 uint8_t username_detected:1, username_found:1, 1140 password_detected:1, password_found:1, 1141 pad:4; 1142 uint8_t character_id; 1143 char username[32], password[32]; 1144 } telnet; 1145 1146 struct { 1147 char answer[96]; 1148 } mdns; 1149 1150 struct { 1151 char version[32]; 1152 } ubntac2; 1153 1154 struct { 1155 /* Via HTTP X-Forwarded-For */ 1156 uint8_t nat_ip[24]; 1157 } http; 1158 1159 struct { 1160 uint8_t auth_found:1, auth_failed:1, _pad:5; 1161 char username[16], password[16]; 1162 } ftp_imap_pop_smtp; 1163 1164 struct { 1165 /* Bittorrent hash */ 1166 uint8_t hash[20]; 1167 } bittorrent; 1168 1169 struct { 1170 char fingerprint[48]; 1171 char class_ident[48]; 1172 } dhcp; 1173 } protos; 1174 1175 /*** ALL protocol specific 64 bit variables here ***/ 1176 1177 /* protocols which have marked a connection as this connection cannot be protocol XXX, multiple uint64_t */ 1178 NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask; 1179 1180 ndpi_protocol_category_t category; 1181 1182 /* NDPI_PROTOCOL_REDIS */ 1183 uint8_t redis_s2d_first_char, redis_d2s_first_char; 1184 1185 uint16_t packet_counter; // can be 0 - 65000 1186 uint16_t packet_direction_counter[2]; 1187 uint16_t byte_counter[2]; 1188 /* NDPI_PROTOCOL_BITTORRENT */ 1189 uint8_t bittorrent_stage; // can be 0 - 255 1190 1191 /* NDPI_PROTOCOL_DIRECTCONNECT */ 1192 uint8_t directconnect_stage:2; // 0 - 1 1193 1194 /* NDPI_PROTOCOL_YAHOO */ 1195 uint8_t sip_yahoo_voice:1; 1196 1197 /* NDPI_PROTOCOL_HTTP */ 1198 uint8_t http_detected:1; 1199 1200 /* NDPI_PROTOCOL_RTSP */ 1201 uint8_t rtsprdt_stage:2, rtsp_control_flow:1; 1202 1203 /* NDPI_PROTOCOL_YAHOO */ 1204 uint8_t yahoo_detection_finished:2; 1205 1206 /* NDPI_PROTOCOL_ZATTOO */ 1207 uint8_t zattoo_stage:3; 1208 1209 /* NDPI_PROTOCOL_QQ */ 1210 uint8_t qq_stage:3; 1211 1212 /* NDPI_PROTOCOL_THUNDER */ 1213 uint8_t thunder_stage:2; // 0 - 3 1214 1215 /* NDPI_PROTOCOL_FLORENSIA */ 1216 uint8_t florensia_stage:1; 1217 1218 /* NDPI_PROTOCOL_SOCKS */ 1219 uint8_t socks5_stage:2, socks4_stage:2; // 0 - 3 1220 1221 /* NDPI_PROTOCOL_EDONKEY */ 1222 uint8_t edonkey_stage:2; // 0 - 3 1223 1224 /* NDPI_PROTOCOL_FTP_CONTROL */ 1225 uint8_t ftp_control_stage:2; 1226 1227 /* NDPI_PROTOCOL_RTMP */ 1228 uint8_t rtmp_stage:2; 1229 1230 /* NDPI_PROTOCOL_PANDO */ 1231 uint8_t pando_stage:3; 1232 1233 /* NDPI_PROTOCOL_STEAM */ 1234 uint16_t steam_stage:3, steam_stage1:3, steam_stage2:2, steam_stage3:2; 1235 1236 /* NDPI_PROTOCOL_PPLIVE */ 1237 uint8_t pplive_stage1:3, pplive_stage2:2, pplive_stage3:2; 1238 1239 /* NDPI_PROTOCOL_STARCRAFT */ 1240 uint8_t starcraft_udp_stage : 3; // 0-7 1241 1242 /* NDPI_PROTOCOL_OPENVPN */ 1243 uint8_t ovpn_session_id[8]; 1244 uint8_t ovpn_counter; 1245 1246 /* NDPI_PROTOCOL_TINC */ 1247 uint8_t tinc_state; 1248 struct tinc_cache_entry tinc_cache_entry; 1249 1250 /* NDPI_PROTOCOL_CSGO */ 1251 uint8_t csgo_strid[18],csgo_state,csgo_s2; 1252 uint32_t csgo_id2; 1253 /* internal structures to save functions calls */ 1254 struct ndpi_packet_struct packet; 1255 struct ndpi_flow_struct *flow; 1256 struct ndpi_id_struct *src; 1257 struct ndpi_id_struct *dst; 1258}; 1259 1260typedef struct { 1261 char *string_to_match, *proto_name; 1262 int protocol_id; 1263 ndpi_protocol_category_t protocol_category; 1264 ndpi_protocol_breed_t protocol_breed; 1265} ndpi_protocol_match; 1266 1267typedef struct { 1268 char *string_to_match; 1269 ndpi_protocol_category_t protocol_category; 1270} ndpi_category_match; 1271 1272typedef struct { 1273 uint32_t network; 1274 uint8_t cidr; 1275 uint8_t value; 1276} ndpi_network; 1277 1278typedef uint32_t ndpi_init_prefs; 1279 1280typedef enum { 1281 ndpi_no_prefs = 0, 1282 ndpi_dont_load_tor_hosts, 1283} ndpi_prefs; 1284 1285typedef struct { 1286 int protocol_id; 1287 ndpi_protocol_category_t protocol_category; 1288 ndpi_protocol_breed_t protocol_breed; 1289} ndpi_protocol_match_result; 1290 1291typedef struct { 1292 char *str; 1293 uint16_t str_len; 1294} ndpi_string; 1295 1296/* **************************************** */ 1297 1298struct ndpi_analyze_struct { 1299 uint32_t *values; 1300 uint32_t min_val, max_val, sum_total, num_data_entries, next_value_insert_index; 1301 uint16_t num_values_array_len /* lenght of the values array */; 1302 1303 struct { 1304 float mu, q; 1305 } stddev; 1306}; 1307 1308#define DEFAULT_SERIES_LEN 64 1309#define MAX_SERIES_LEN 512 1310#define MIN_SERIES_LEN 8 1311 1312typedef struct ndpi_ptree ndpi_ptree_t; 1313 1314""" 1315 1316cc_ndpi_apis = """ 1317struct ndpi_detection_module_struct *ndpi_init_detection_module(void); 1318void *memset(void *str, int c, size_t n); 1319void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct, 1320 const NDPI_PROTOCOL_BITMASK * detection_bitmask); 1321ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct, 1322 struct ndpi_flow_struct *flow, 1323 const unsigned char *packet, 1324 const unsigned short packetlen, 1325 const uint64_t current_tick, 1326 struct ndpi_id_struct *src, 1327 struct ndpi_id_struct *dst); 1328ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, 1329 struct ndpi_flow_struct *flow, 1330 uint8_t enable_guess, 1331 uint8_t *protocol_was_guessed); 1332 1333void * ndpi_malloc(size_t size); 1334void ndpi_free(void *ptr); 1335void * ndpi_flow_malloc(size_t size); 1336void ndpi_flow_free(void *ptr); 1337void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct); 1338char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, 1339 ndpi_protocol proto, 1340 char *buf, unsigned buf_len); 1341const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category); 1342char* ndpi_revision(void); 1343void ndpi_finalize_initalization(struct ndpi_detection_module_struct *ndpi_str); 1344uint32_t ndpi_detection_get_sizeof_ndpi_flow_struct(void); 1345uint32_t ndpi_detection_get_sizeof_ndpi_id_struct(void); 1346uint32_t ndpi_detection_get_sizeof_ndpi_flow_tcp_struct(void); 1347uint32_t ndpi_detection_get_sizeof_ndpi_flow_udp_struct(void); 1348uint8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); 1349""" 1350 1351 1352def check_structures_size(flow_struct_defined, flow_struct_loaded, 1353 id_struct_defined, id_struct_loaded, 1354 tcp_flow_struct_defined, tcp_flow_struct_loaded, 1355 udp_flow_struct_defined, udp_flow_struct_loaded): 1356 """ Function used to check loaded structures sizes againt defined ones """ 1357 errors = [] 1358 if flow_struct_defined != flow_struct_loaded: 1359 errors.append('ndpi_flow_struct') 1360 if id_struct_defined != id_struct_loaded: 1361 errors.append('ndpi_id_struct') 1362 if tcp_flow_struct_defined != tcp_flow_struct_loaded: 1363 errors.append('ndpi_tcp_flow_struct') 1364 if udp_flow_struct_defined != udp_flow_struct_loaded: 1365 errors.append('ndpi_udp_flow_struct') 1366 return errors 1367 1368 1369class NDPI(): 1370 """ ndpi module main class """ 1371 1372 def __init__(self, libpath=None, max_tcp_dissections=80, max_udp_dissections=16, enable_guess=True): 1373 self._ffi = cffi.FFI() 1374 if libpath is None: 1375 self._ndpi = self._ffi.dlopen(dirname(abspath(__file__)) + '/libndpi.so') 1376 else: 1377 self._ndpi = self._ffi.dlopen(libpath) 1378 self._ffi.cdef(cc_ndpi_network_headers, packed=True) 1379 self._ffi.cdef(cc_ndpi_stuctures) 1380 self._ffi.cdef(cc_ndpi_apis) 1381 self._mod = self._ndpi.ndpi_init_detection_module() 1382 ndpi_revision = self._ffi.string(self._ndpi.ndpi_revision()).decode('utf-8', errors='ignore') 1383 if ndpi_revision[:3] >= '3.1': 1384 self._ndpi.ndpi_finalize_initalization(self._mod) 1385 all = self._ffi.new('NDPI_PROTOCOL_BITMASK*') 1386 self._ndpi.memset(self._ffi.cast("char *", all), 0xFF, self._ffi.sizeof("NDPI_PROTOCOL_BITMASK")) 1387 self._ndpi.ndpi_set_protocol_detection_bitmask2(self._mod, all) 1388 errors = check_structures_size(self._ffi.sizeof("struct ndpi_flow_struct"), 1389 self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_struct(), 1390 self._ffi.sizeof("struct ndpi_id_struct"), 1391 self._ndpi.ndpi_detection_get_sizeof_ndpi_id_struct(), 1392 self._ffi.sizeof("struct ndpi_flow_tcp_struct"), 1393 self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_tcp_struct(), 1394 self._ffi.sizeof("struct ndpi_flow_udp_struct"), 1395 self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_udp_struct()) 1396 if len(errors) != 0: 1397 raise ValueError('nDPI error: mismatch in the headers of following structures{}'.format(', '.join(errors))) 1398 else: 1399 self.SIZEOF_FLOW_STRUCT = self._ffi.sizeof("struct ndpi_flow_struct") 1400 self.SIZEOF_ID_STRUCT = self._ffi.sizeof("struct ndpi_id_struct") 1401 self.NULL = self._ffi.NULL 1402 self.max_tcp_dissections = max_tcp_dissections 1403 self.max_udp_dissections = max_udp_dissections 1404 self.enable_guess = enable_guess 1405 1406 def new_ndpi_flow(self): 1407 """ Create a new nDPI flow object """ 1408 f = self._ffi.cast('struct ndpi_flow_struct*', self._ndpi.ndpi_flow_malloc(self.SIZEOF_FLOW_STRUCT)) 1409 self._ndpi.memset(f, 0, self.SIZEOF_FLOW_STRUCT) 1410 return f 1411 1412 def new_ndpi_id(self): 1413 """ Create a new nDPI id object """ 1414 i = self._ffi.cast('struct ndpi_id_struct*', self._ndpi.ndpi_malloc(self.SIZEOF_ID_STRUCT)) 1415 self._ndpi.memset(i, 0, self.SIZEOF_ID_STRUCT) 1416 return i 1417 1418 def ndpi_detection_process_packet(self, flow, packet, packetlen, current_tick, src, dst): 1419 """ Main detection processing function """ 1420 p = self._ndpi.ndpi_detection_process_packet(self._mod, flow, packet, packetlen, current_tick, src, dst) 1421 return p 1422 1423 def ndpi_detection_giveup(self, flow): 1424 """ Giveup detection function """ 1425 return self._ndpi.ndpi_detection_giveup(self._mod, flow, self.enable_guess, self._ffi.new("uint8_t*", 0)) 1426 1427 def ndpi_flow_free(self, flow): 1428 """ Free nDPI flow object """ 1429 return self._ndpi.ndpi_flow_free(flow) 1430 1431 def ndpi_free(self, ptr): 1432 """ Free nDPI object """ 1433 return self._ndpi.ndpi_free(ptr) 1434 1435 def get_str_field(self, ptr): 1436 """ Get fixed string size attribute """ 1437 if ptr == self._ffi.NULL: 1438 return '' 1439 else: 1440 return self._ffi.string(ptr).decode('utf-8', errors='ignore') 1441 1442 def get_buffer_field(self, ptr, li): 1443 """ Get variable string size attribute """ 1444 if ptr == self._ffi.NULL: 1445 return '' 1446 else: 1447 return self._ffi.string(ptr, li).decode('utf-8', errors='ignore') 1448 1449 def ndpi_protocol2name(self, proto): 1450 """ Convert nDPI protocol object to readable name """ 1451 buf = self._ffi.new("char[32]") 1452 self._ndpi.ndpi_protocol2name(self._mod, proto, buf, self._ffi.sizeof(buf)) 1453 return self._ffi.string(buf).decode('utf-8', errors='ignore') 1454 1455 def ndpi_category_get_name(self, category): 1456 """ Convert nDPI protocol object to readable name """ 1457 return self._ffi.string(self._ndpi.ndpi_category_get_name(self._mod, category)).decode('utf-8', errors='ignore') 1458 1459 def ndpi_extra_dissection_possible(self, flow): 1460 return self._ndpi.ndpi_extra_dissection_possible(self._mod, flow) 1461 1462 def ndpi_exit_detection_module(self): 1463 """ Exit function for nDPI module """ 1464 self._ndpi.ndpi_exit_detection_module(self._mod) 1465 self._ffi.dlclose(self._ndpi) 1466