1 /* 2 pmacct (Promiscuous mode IP Accounting package) 3 pmacct is Copyright (C) 2003-2019 by Paolo Lucente 4 */ 5 6 /* 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 2 of the License, or 10 (at your option) any later version. 11 12 This program is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License 18 along with this program; if not, write to the Free Software 19 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 */ 21 22 #ifndef SQL_COMMON_H 23 #define SQL_COMMON_H 24 25 /* includes */ 26 #include <sys/poll.h> 27 #include "net_aggr.h" 28 #include "ports_aggr.h" 29 30 /* defines */ 31 #define DEFAULT_DB_REFRESH_TIME 60 32 #define DEFAULT_SQL_TABLE_VERSION 1 33 #define DEFAULT_SQL_WRITERS_NO 10 34 #define CACHE_ENTRIES 32771 35 #define QUERY_BUFFER_SIZE 32768 36 #define MAGIC 14021979 37 #define DEF_HDR_FIELD_LEN 128 38 #define MAX_LOGFILE_SIZE 2048000000 39 #define MAX_LOGFILE_ROTATIONS 1000 40 41 /* cache elements defines */ 42 #define REASONABLE_NUMBER 100 43 #define STALE_M 3 44 #define RETIRE_M STALE_M*STALE_M 45 46 /* backend types */ 47 #define BE_TYPE_PRIMARY 0 48 #define BE_TYPE_BACKUP 1 49 #define BE_TYPE_LOGFILE 2 50 51 /* lock types */ 52 #define PM_LOCK_EXCLUSIVE 0 53 #define PM_LOCK_ROW_EXCLUSIVE 1 54 #define PM_LOCK_NONE 2 55 56 /* cache element states */ 57 #define SQL_CACHE_FREE 0 58 #define SQL_CACHE_COMMITTED 1 59 #define SQL_CACHE_INUSE 2 60 #define SQL_CACHE_INVALID 3 61 #define SQL_CACHE_ERROR 255 62 63 #define SQL_TABLE_VERSION_PLAIN 0 64 #define SQL_TABLE_VERSION_BGP 1000 65 66 /* macros */ 67 #define SPACELEFT(x) (sizeof(x)-strlen(x)-1) 68 #define SPACELEFT_LEN(x,y) (sizeof(x)-y) 69 #define SPACELEFT_PTR(x,y) (y-strlen(x)) 70 71 #define SQL_INSERT_INSERT 0x00000001 72 #define SQL_INSERT_UPDATE 0x00000002 73 #define SQL_INSERT_PRO_RATING 0x00000004 74 #define SQL_INSERT_SAFE_ACTION 0x00000008 75 76 struct multi_values { 77 int buffer_offset; /* multi-values buffer offset where to write next query */ 78 int head_buffer_elem; /* first multi-values buffer element */ 79 int buffer_elem_num; /* number of elements in the multi-values buffer */ 80 int last_queue_elem; /* last queue element signallation */ 81 }; 82 83 /* structures */ 84 struct insert_data { 85 struct configuration *cfg; 86 unsigned int hash; 87 unsigned int modulo; 88 time_t now; 89 time_t basetime; 90 time_t triggertime; 91 time_t timeslot; /* counters timeslot */ 92 time_t t_timeslot; /* trigger timeslot */ 93 struct timeval flushtime; /* last time the table has been flushed */ 94 int pending_accumulators; 95 int num_primitives; 96 int dyn_table; 97 int dyn_table_time_only; 98 char dyn_table_name[SRVBUFLEN]; 99 int recover; 100 int locks; 101 time_t new_basetime; 102 time_t committed_basetime; 103 int current_queue_elem; 104 struct multi_values mv; 105 int cp_idx; /* custom primitives index */ 106 /* stats */ 107 time_t elap_time; /* elapsed time */ 108 unsigned int ten; /* total elements number */ 109 unsigned int een; /* effective elements number */ 110 unsigned int qn; /* total query number */ 111 unsigned int iqn; /* INSERTs query number */ 112 unsigned int uqn; /* UPDATEs query number */ 113 }; 114 115 struct db_cache { 116 struct pkt_primitives primitives; 117 pm_counter_t bytes_counter; 118 pm_counter_t packet_counter; 119 pm_counter_t flows_counter; 120 u_int8_t flow_type; 121 u_int32_t tcp_flags; 122 u_int8_t tentatives; /* support to classifiers: tentatives remaining */ 123 time_t basetime; 124 struct pkt_bgp_primitives *pbgp; 125 struct pkt_nat_primitives *pnat; 126 struct pkt_mpls_primitives *pmpls; 127 struct pkt_tunnel_primitives *ptun; 128 u_char *pcust; 129 struct pkt_vlen_hdr_primitives *pvlen; 130 u_int8_t valid; 131 u_int8_t prep_valid; 132 unsigned int signature; 133 u_int8_t chained; 134 struct pkt_stitching *stitch; 135 struct db_cache *prev; 136 struct db_cache *next; 137 time_t start_tag; /* time: first packet received */ 138 time_t lru_tag; /* time: last packet received */ 139 struct db_cache *lru_prev; 140 struct db_cache *lru_next; 141 }; 142 143 typedef void (*dbop_handler) (const struct db_cache *, struct insert_data *, int, char **, char **); 144 145 struct frags { 146 dbop_handler handler; 147 u_int64_t type; 148 char string[SRVBUFLEN]; 149 }; 150 151 /* Backend descriptors */ 152 struct DBdesc { 153 void *desc; 154 char *conn_string; /* PostgreSQL */ 155 char *filename; /* SQLite */ 156 char *errmsg; 157 short int type; 158 short int connected; 159 short int fail; 160 }; 161 162 struct BE_descs { 163 struct DBdesc *p; 164 struct DBdesc *b; 165 }; 166 167 /* Callbacks for a common SQL layer */ 168 typedef void (*db_connect)(struct DBdesc *, char *); 169 typedef void (*db_close)(struct BE_descs *); 170 typedef void (*db_lock)(struct DBdesc *); 171 typedef void (*db_unlock)(struct BE_descs *); 172 typedef void (*db_create_table)(struct DBdesc *, char *); 173 typedef int (*db_op)(struct DBdesc *, struct db_cache *, struct insert_data *); 174 typedef void (*sqlcache_purge)(struct db_cache *[], int, struct insert_data *); 175 typedef void (*sqlbackend_create)(struct DBdesc *); 176 struct sqlfunc_cb_registry { 177 db_connect connect; 178 db_close close; 179 db_lock lock; 180 db_unlock unlock; 181 db_op op; 182 db_create_table create_table; 183 sqlbackend_create create_backend; 184 sqlcache_purge purge; 185 /* flush and query wrapper are common for all SQL plugins */ 186 }; 187 188 189 #include "preprocess.h" 190 191 /* functions */ 192 extern void count_src_mac_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 193 extern void count_dst_mac_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 194 extern void count_vlan_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 195 extern void count_cos_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 196 extern void count_etype_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 197 extern void count_src_host_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 198 extern void count_src_net_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 199 extern void count_src_as_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 200 extern void count_dst_host_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 201 extern void count_dst_net_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 202 extern void count_dst_as_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 203 extern void count_std_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 204 extern void count_ext_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 205 extern void count_lrg_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 206 extern void count_as_path_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 207 extern void count_local_pref_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 208 extern void count_med_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 209 extern void count_dst_roa_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 210 extern void count_src_std_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 211 extern void count_src_ext_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 212 extern void count_src_lrg_comm_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 213 extern void count_src_as_path_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 214 extern void count_src_local_pref_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 215 extern void count_src_med_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 216 extern void count_src_roa_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 217 extern void count_mpls_vpn_rd_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 218 extern void count_mpls_pw_id_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 219 extern void count_peer_src_as_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 220 extern void count_peer_dst_as_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 221 extern void count_peer_src_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 222 extern void count_peer_dst_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 223 extern void count_src_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 224 extern void count_dst_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 225 extern void count_ip_tos_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 226 extern void count_in_iface_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 227 extern void count_out_iface_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 228 extern void count_src_nmask_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 229 extern void count_dst_nmask_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 230 extern void count_sampling_rate_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 231 extern void count_sampling_direction_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 232 extern void MY_count_ip_proto_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 233 extern void PG_count_ip_proto_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 234 extern void count_timestamp_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 235 extern void count_copy_timestamp_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 236 extern void count_tag_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 237 extern void count_tag2_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 238 extern void count_label_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 239 extern void count_class_id_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 240 extern void count_tcpflags_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 241 extern void count_post_nat_src_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 242 extern void count_post_nat_dst_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 243 extern void count_post_nat_src_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 244 extern void count_post_nat_dst_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 245 extern void count_nat_event_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 246 extern void count_mpls_label_top_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 247 extern void count_mpls_label_bottom_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 248 extern void count_mpls_stack_depth_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 249 extern void count_tunnel_src_mac_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 250 extern void count_tunnel_dst_mac_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 251 extern void count_tunnel_src_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 252 extern void count_tunnel_dst_ip_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 253 extern void MY_count_tunnel_ip_proto_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 254 extern void PG_count_tunnel_ip_proto_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 255 extern void count_tunnel_ip_tos_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 256 extern void count_tunnel_src_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 257 extern void count_tunnel_dst_port_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 258 extern void count_vxlan_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 259 extern void count_timestamp_start_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 260 extern void PG_copy_count_timestamp_start_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 261 extern void count_timestamp_start_residual_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 262 extern void count_timestamp_end_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 263 extern void PG_copy_count_timestamp_end_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 264 extern void count_timestamp_end_residual_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 265 extern void count_timestamp_arrival_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 266 extern void PG_copy_count_timestamp_arrival_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 267 extern void count_timestamp_arrival_residual_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 268 extern void count_timestamp_min_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 269 extern void PG_copy_count_timestamp_min_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 270 extern void count_timestamp_min_residual_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 271 extern void count_timestamp_max_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 272 extern void PG_copy_count_timestamp_max_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 273 extern void count_export_proto_seqno_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 274 extern void count_export_proto_version_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 275 extern void count_export_proto_sysid_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 276 extern void count_timestamp_max_residual_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 277 extern void count_custom_primitives_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 278 extern void fake_mac_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 279 extern void fake_host_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 280 extern void fake_as_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 281 extern void fake_comms_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 282 extern void fake_as_path_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 283 extern void fake_mpls_vpn_rd_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 284 285 extern void count_src_host_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 286 extern void count_dst_host_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 287 extern void count_src_net_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 288 extern void count_dst_net_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 289 extern void count_peer_src_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 290 extern void count_peer_dst_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 291 extern void count_post_nat_src_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 292 extern void count_post_nat_dst_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 293 extern void count_tunnel_src_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 294 extern void count_tunnel_dst_ip_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 295 extern void fake_host_aton_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 296 297 #if defined (WITH_GEOIP) || defined (WITH_GEOIPV2) 298 extern void count_src_host_country_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 299 extern void count_dst_host_country_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 300 #endif 301 #if defined (WITH_GEOIPV2) 302 extern void count_src_host_pocode_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 303 extern void count_dst_host_pocode_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 304 extern void count_src_host_coords_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 305 extern void count_dst_host_coords_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 306 #endif 307 308 #if defined (WITH_NDPI) 309 extern void count_ndpi_class_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 310 #endif 311 312 extern void count_counters_setclause_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 313 extern void count_flows_setclause_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 314 extern void count_tcpflags_setclause_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 315 extern void count_noop_setclause_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 316 extern void count_noop_setclause_event_handler(const struct db_cache *, struct insert_data *, int, char **, char **); 317 318 /* Toward a common SQL layer */ 319 extern void sql_set_signals(); 320 extern void sql_set_insert_func(); 321 extern void sql_init_maps(struct extra_primitives *, struct primitives_ptrs *, struct networks_table *, struct networks_cache *, struct ports_table *); 322 extern void sql_init_global_buffers(); 323 extern void sql_init_default_values(struct extra_primitives *); 324 extern void sql_init_historical_acct(time_t, struct insert_data *); 325 extern void sql_init_triggers(time_t, struct insert_data *); 326 extern void sql_init_refresh_deadline(time_t *); 327 extern void sql_link_backend_descriptors(struct BE_descs *, struct DBdesc *, struct DBdesc *); 328 extern void sql_cache_modulo(struct primitives_ptrs *, struct insert_data *); 329 extern int sql_cache_flush(struct db_cache *[], int, struct insert_data *, int); 330 extern void sql_cache_flush_pending(struct db_cache *[], int, struct insert_data *); 331 extern void sql_cache_handle_flush_event(struct insert_data *, time_t *, struct ports_table *); 332 extern void sql_cache_insert(struct primitives_ptrs *, struct insert_data *); 333 extern struct db_cache *sql_cache_search(struct primitives_ptrs *, time_t); 334 extern int sql_trigger_exec(char *); 335 extern void sql_db_ok(struct DBdesc *); 336 extern void sql_db_fail(struct DBdesc *); 337 extern void sql_db_errmsg(struct DBdesc *); 338 extern void sql_db_warnmsg(struct DBdesc *); 339 extern int sql_query(struct BE_descs *, struct db_cache *, struct insert_data *); 340 extern void sql_exit_gracefully(int); 341 extern int sql_evaluate_primitives(int); 342 extern void sql_create_table(struct DBdesc *, time_t *, struct primitives_ptrs *); 343 extern void sql_invalidate_shadow_entries(struct db_cache *[], int *); 344 extern int sql_select_locking_style(char *); 345 extern int sql_compose_static_set(int); 346 extern int sql_compose_static_set_event(); 347 extern void primptrs_set_all_from_db_cache(struct primitives_ptrs *, struct db_cache *); 348 349 extern void sql_sum_host_insert(struct primitives_ptrs *, struct insert_data *); 350 extern void sql_sum_port_insert(struct primitives_ptrs *, struct insert_data *); 351 extern void sql_sum_as_insert(struct primitives_ptrs *, struct insert_data *); 352 #if defined (HAVE_L2) 353 extern void sql_sum_mac_insert(struct primitives_ptrs *, struct insert_data *); 354 #endif 355 extern void sql_sum_std_comm_insert(struct primitives_ptrs *, struct insert_data *); 356 extern void sql_sum_ext_comm_insert(struct primitives_ptrs *, struct insert_data *); 357 358 #if 10 359 /* Global Variables: a simple way of gain precious speed when playing with strings */ 360 extern char sql_data[LARGEBUFLEN]; 361 extern char lock_clause[LONGSRVBUFLEN]; 362 extern char unlock_clause[LONGSRVBUFLEN]; 363 extern char update_clause[LONGSRVBUFLEN]; 364 extern char set_clause[LONGSRVBUFLEN]; 365 extern char copy_clause[LONGSRVBUFLEN]; 366 extern char insert_clause[LONGSRVBUFLEN]; 367 extern char insert_counters_clause[LONGSRVBUFLEN]; 368 extern char insert_nocounters_clause[LONGSRVBUFLEN]; 369 extern char insert_full_clause[LONGSRVBUFLEN]; 370 extern char values_clause[LONGLONGSRVBUFLEN]; 371 extern char *multi_values_buffer; 372 extern char where_clause[LONGLONGSRVBUFLEN]; 373 extern unsigned char *pipebuf; 374 extern struct db_cache *sql_cache; 375 extern struct db_cache **sql_queries_queue, **sql_pending_queries_queue; 376 extern struct db_cache *collision_queue; 377 extern int cq_ptr, qq_ptr, qq_size, pp_size, pb_size, pn_size, pm_size, pt_size; 378 extern int pc_size, dbc_size, cq_size, pqq_ptr; 379 extern struct db_cache lru_head, *lru_tail; 380 extern struct frags where[N_PRIMITIVES+2]; 381 extern struct frags values[N_PRIMITIVES+2]; 382 extern struct frags copy_values[N_PRIMITIVES+2]; 383 extern struct frags set[N_PRIMITIVES+2]; 384 extern struct frags set_event[N_PRIMITIVES+2]; 385 extern int glob_num_primitives; /* last resort for signal handling */ 386 extern int glob_basetime; /* last resort for signal handling */ 387 extern time_t glob_new_basetime; /* last resort for signal handling */ 388 extern time_t glob_committed_basetime; /* last resort for signal handling */ 389 extern int glob_dyn_table, glob_dyn_table_time_only; /* last resort for signal handling */ 390 extern int glob_timeslot; /* last resort for sql handlers */ 391 392 extern struct sqlfunc_cb_registry sqlfunc_cbr; 393 extern void (*insert_func)(struct primitives_ptrs *, struct insert_data *); 394 extern struct DBdesc p; 395 extern struct DBdesc b; 396 extern struct BE_descs bed; 397 extern struct largebuf_s envbuf; 398 extern time_t now; /* PostgreSQL */ 399 #endif 400 #endif //SQL_COMMON_H 401