1 /*- 2 * Copyright 2016 Vsevolod Stakhov 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef RSPAMD_SYMBOLS_CACHE_H 17 #define RSPAMD_SYMBOLS_CACHE_H 18 19 #include "config.h" 20 #include "ucl.h" 21 #include "cfg_file.h" 22 #include "contrib/libev/ev.h" 23 24 #include <lua.h> 25 26 #ifdef __cplusplus 27 extern "C" { 28 #endif 29 30 struct rspamd_task; 31 struct rspamd_config; 32 struct rspamd_symcache; 33 struct rspamd_worker; 34 struct rspamd_symcache_item; 35 struct rspamd_config_settings_elt; 36 37 typedef void (*symbol_func_t) (struct rspamd_task *task, 38 struct rspamd_symcache_item *item, 39 gpointer user_data); 40 41 enum rspamd_symbol_type { 42 SYMBOL_TYPE_NORMAL = (1u << 0u), 43 SYMBOL_TYPE_VIRTUAL = (1u << 1u), 44 SYMBOL_TYPE_CALLBACK = (1u << 2u), 45 SYMBOL_TYPE_GHOST = (1u << 3u), 46 SYMBOL_TYPE_SKIPPED = (1u << 4u), 47 SYMBOL_TYPE_COMPOSITE = (1u << 5u), 48 SYMBOL_TYPE_CLASSIFIER = (1u << 6u), 49 SYMBOL_TYPE_FINE = (1u << 7u), 50 SYMBOL_TYPE_EMPTY = (1u << 8u), /* Allow execution on empty tasks */ 51 SYMBOL_TYPE_CONNFILTER = (1u << 9u), /* Connection stage filter */ 52 SYMBOL_TYPE_PREFILTER = (1u << 10u), 53 SYMBOL_TYPE_POSTFILTER = (1u << 11u), 54 SYMBOL_TYPE_NOSTAT = (1u << 12u), /* Skip as statistical symbol */ 55 SYMBOL_TYPE_IDEMPOTENT = (1u << 13u), /* Symbol cannot change metric */ 56 SYMBOL_TYPE_TRIVIAL = (1u << 14u), /* Symbol is trivial */ 57 SYMBOL_TYPE_MIME_ONLY = (1u << 15u), /* Symbol is mime only */ 58 SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u), /* Symbol should be disabled explicitly only */ 59 SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1u << 17u), /* Symbol ignores passthrough result */ 60 SYMBOL_TYPE_EXPLICIT_ENABLE = (1u << 18u), /* Symbol should be enabled explicitly only */ 61 SYMBOL_TYPE_USE_CORO = (1u << 19u), /* Symbol uses lua coroutines */ 62 }; 63 64 /** 65 * Abstract structure for saving callback data for symbols 66 */ 67 struct rspamd_abstract_callback_data { 68 guint64 magic; 69 char data[]; 70 }; 71 72 struct rspamd_symcache_item_stat { 73 struct rspamd_counter_data time_counter; 74 gdouble avg_time; 75 gdouble weight; 76 guint hits; 77 guint64 total_hits; 78 struct rspamd_counter_data frequency_counter; 79 gdouble avg_frequency; 80 gdouble stddev_frequency; 81 }; 82 83 /** 84 * Creates new cache structure 85 * @return 86 */ 87 struct rspamd_symcache *rspamd_symcache_new (struct rspamd_config *cfg); 88 89 /** 90 * Remove the cache structure syncing data if needed 91 * @param cache 92 */ 93 void rspamd_symcache_destroy (struct rspamd_symcache *cache); 94 95 /** 96 * Saves symbols cache to disk if possible 97 * @param cache 98 */ 99 void rspamd_symcache_save (struct rspamd_symcache *cache); 100 101 /** 102 * Load symbols cache from file, must be called _after_ init_symbols_cache 103 */ 104 gboolean rspamd_symcache_init (struct rspamd_symcache *cache); 105 106 /** 107 * Generic function to register a symbol 108 * @param cache 109 * @param name 110 * @param weight 111 * @param priority 112 * @param func 113 * @param user_data 114 * @param type 115 * @param parent 116 */ 117 gint rspamd_symcache_add_symbol (struct rspamd_symcache *cache, 118 const gchar *name, 119 gint priority, 120 symbol_func_t func, 121 gpointer user_data, 122 enum rspamd_symbol_type type, 123 gint parent); 124 125 /** 126 * Add callback to be executed whenever symbol has peak value 127 * @param cache 128 * @param cbref 129 */ 130 void rspamd_symcache_set_peak_callback (struct rspamd_symcache *cache, 131 gint cbref); 132 133 /** 134 * Add delayed condition to the specific symbol in cache. So symbol can be absent 135 * to the moment of addition 136 * @param cache 137 * @param id id of symbol 138 * @param L lua state pointer 139 * @param cbref callback reference (returned by luaL_ref) 140 * @return TRUE if condition has been added 141 */ 142 gboolean rspamd_symcache_add_condition_delayed (struct rspamd_symcache *cache, 143 const gchar *sym, 144 lua_State *L, gint cbref); 145 146 /** 147 * Find symbol in cache by id and returns its id resolving virtual symbols if 148 * applicable 149 * @param cache 150 * @param name 151 * @return id of symbol or (-1) if a symbol has not been found 152 */ 153 gint rspamd_symcache_find_symbol (struct rspamd_symcache *cache, 154 const gchar *name); 155 156 /** 157 * Get statistics for a specific symbol 158 * @param cache 159 * @param name 160 * @param frequency 161 * @param tm 162 * @return 163 */ 164 gboolean rspamd_symcache_stat_symbol (struct rspamd_symcache *cache, 165 const gchar *name, 166 gdouble *frequency, 167 gdouble *freq_stddev, 168 gdouble *tm, 169 guint *nhits); 170 171 /** 172 * Find symbol in cache by its id 173 * @param cache 174 * @param id 175 * @return symbol's name or NULL 176 */ 177 const gchar *rspamd_symcache_symbol_by_id (struct rspamd_symcache *cache, 178 gint id); 179 180 /** 181 * Returns number of symbols registered in symbols cache 182 * @param cache 183 * @return number of symbols in the cache 184 */ 185 guint rspamd_symcache_stats_symbols_count (struct rspamd_symcache *cache); 186 187 /** 188 * Call function for cached symbol using saved callback 189 * @param task task object 190 * @param cache symbols cache 191 * @param saved_item pointer to currently saved item 192 */ 193 gboolean rspamd_symcache_process_symbols (struct rspamd_task *task, 194 struct rspamd_symcache *cache, 195 gint stage); 196 197 /** 198 * Validate cache items against theirs weights defined in metrics 199 * @param cache symbols cache 200 * @param cfg configuration 201 * @param strict do strict checks - symbols MUST be described in metrics 202 */ 203 gboolean rspamd_symcache_validate (struct rspamd_symcache *cache, 204 struct rspamd_config *cfg, 205 gboolean strict); 206 207 /** 208 * Return statistics about the cache as ucl object (array of objects one per item) 209 * @param cache 210 * @return 211 */ 212 ucl_object_t *rspamd_symcache_counters (struct rspamd_symcache *cache); 213 214 /** 215 * Start cache reloading 216 * @param cache 217 * @param ev_base 218 */ 219 void rspamd_symcache_start_refresh (struct rspamd_symcache *cache, 220 struct ev_loop *ev_base, 221 struct rspamd_worker *w); 222 223 /** 224 * Increases counter for a specific symbol 225 * @param cache 226 * @param symbol 227 */ 228 void rspamd_symcache_inc_frequency (struct rspamd_symcache *cache, 229 struct rspamd_symcache_item *item); 230 231 /** 232 * Add dependency relation between two symbols identified by id (source) and 233 * a symbolic name (destination). Destination could be virtual or real symbol. 234 * Callback destinations are not yet supported. 235 * @param id_from source symbol 236 * @param to destination name 237 */ 238 void rspamd_symcache_add_dependency (struct rspamd_symcache *cache, 239 gint id_from, const gchar *to, 240 gint virtual_id_from); 241 242 /** 243 * Add delayed dependency that is resolved on cache post-load routine 244 * @param cache 245 * @param from 246 * @param to 247 */ 248 void rspamd_symcache_add_delayed_dependency (struct rspamd_symcache *cache, 249 const gchar *from, const gchar *to); 250 251 /** 252 * Disable specific symbol in the cache 253 * @param cache 254 * @param symbol 255 */ 256 void rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache, 257 const gchar *symbol, 258 gboolean resolve_parent); 259 260 /** 261 * Enable specific symbol in the cache 262 * @param cache 263 * @param symbol 264 */ 265 void rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache, 266 const gchar *symbol); 267 268 /** 269 * Get abstract callback data for a symbol (or its parent symbol) 270 * @param cache cache object 271 * @param symbol symbol name 272 * @return abstract callback data or NULL if symbol is absent or has no data attached 273 */ 274 struct rspamd_abstract_callback_data *rspamd_symcache_get_cbdata ( 275 struct rspamd_symcache *cache, const gchar *symbol); 276 277 /** 278 * Returns symbol's parent name (or symbol name itself) 279 * @param cache 280 * @param symbol 281 * @return 282 */ 283 const gchar *rspamd_symcache_get_parent (struct rspamd_symcache *cache, 284 const gchar *symbol); 285 286 /** 287 * Adds flags to a symbol 288 * @param cache 289 * @param symbol 290 * @param flags 291 * @return 292 */ 293 gboolean rspamd_symcache_add_symbol_flags (struct rspamd_symcache *cache, 294 const gchar *symbol, 295 guint flags); 296 297 gboolean rspamd_symcache_set_symbol_flags (struct rspamd_symcache *cache, 298 const gchar *symbol, 299 guint flags); 300 301 guint rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache, 302 const gchar *symbol); 303 304 /** 305 * Process settings for task 306 * @param task 307 * @param cache 308 * @return 309 */ 310 gboolean rspamd_symcache_process_settings (struct rspamd_task *task, 311 struct rspamd_symcache *cache); 312 313 314 /** 315 * Checks if a symbol specified has been checked (or disabled) 316 * @param task 317 * @param cache 318 * @param symbol 319 * @return 320 */ 321 gboolean rspamd_symcache_is_checked (struct rspamd_task *task, 322 struct rspamd_symcache *cache, 323 const gchar *symbol); 324 325 /** 326 * Returns checksum for all cache items 327 * @param cache 328 * @return 329 */ 330 guint64 rspamd_symcache_get_cksum (struct rspamd_symcache *cache); 331 332 /** 333 * Checks if a symbols is enabled (not checked and conditions return true if present) 334 * @param task 335 * @param cache 336 * @param symbol 337 * @return 338 */ 339 gboolean rspamd_symcache_is_symbol_enabled (struct rspamd_task *task, 340 struct rspamd_symcache *cache, 341 const gchar *symbol); 342 343 /** 344 * Enable this symbol for task 345 * @param task 346 * @param cache 347 * @param symbol 348 * @return TRUE if a symbol has been enabled (not executed before) 349 */ 350 gboolean rspamd_symcache_enable_symbol (struct rspamd_task *task, 351 struct rspamd_symcache *cache, 352 const gchar *symbol); 353 354 /** 355 * Enable this symbol for task 356 * @param task 357 * @param cache 358 * @param symbol 359 * @return TRUE if a symbol has been disabled (not executed before) 360 */ 361 gboolean rspamd_symcache_disable_symbol (struct rspamd_task *task, 362 struct rspamd_symcache *cache, 363 const gchar *symbol); 364 365 /** 366 * Process specific function for each cache element (in order they are added) 367 * @param cache 368 * @param func 369 * @param ud 370 */ 371 void rspamd_symcache_foreach (struct rspamd_symcache *cache, 372 void (*func) (struct rspamd_symcache_item *item, gpointer /* userdata */), 373 gpointer ud); 374 375 /** 376 * Returns the current item being processed (if any) 377 * @param task 378 * @return 379 */ 380 struct rspamd_symcache_item *rspamd_symcache_get_cur_item (struct rspamd_task *task); 381 382 /** 383 * Replaces the current item being processed. 384 * Returns the current item being processed (if any) 385 * @param task 386 * @param item 387 * @return 388 */ 389 struct rspamd_symcache_item *rspamd_symcache_set_cur_item (struct rspamd_task *task, 390 struct rspamd_symcache_item *item); 391 392 393 /** 394 * Finalize the current async element potentially calling its deps 395 */ 396 void rspamd_symcache_finalize_item (struct rspamd_task *task, 397 struct rspamd_symcache_item *item); 398 399 /* 400 * Increase number of async events pending for an item 401 */ 402 guint rspamd_symcache_item_async_inc_full (struct rspamd_task *task, 403 struct rspamd_symcache_item *item, 404 const gchar *subsystem, 405 const gchar *loc); 406 407 #define rspamd_symcache_item_async_inc(task, item, subsystem) \ 408 rspamd_symcache_item_async_inc_full(task, item, subsystem, G_STRLOC) 409 410 /* 411 * Decrease number of async events pending for an item, asserts if no events pending 412 */ 413 guint rspamd_symcache_item_async_dec_full (struct rspamd_task *task, 414 struct rspamd_symcache_item *item, 415 const gchar *subsystem, 416 const gchar *loc); 417 418 #define rspamd_symcache_item_async_dec(task, item, subsystem) \ 419 rspamd_symcache_item_async_dec_full(task, item, subsystem, G_STRLOC) 420 421 /** 422 * Decrease number of async events pending for an item, asserts if no events pending 423 * If no events are left, this function calls `rspamd_symbols_cache_finalize_item` and returns TRUE 424 * @param task 425 * @param item 426 * @return 427 */ 428 gboolean rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task, 429 struct rspamd_symcache_item *item, 430 const gchar *subsystem, 431 const gchar *loc); 432 433 #define rspamd_symcache_item_async_dec_check(task, item, subsystem) \ 434 rspamd_symcache_item_async_dec_check_full(task, item, subsystem, G_STRLOC) 435 436 /** 437 * Disables execution of all symbols, excluding those specified in `skip_mask` 438 * @param task 439 * @param cache 440 * @param skip_mask 441 */ 442 void rspamd_symcache_disable_all_symbols (struct rspamd_task *task, 443 struct rspamd_symcache *cache, 444 guint skip_mask); 445 446 /** 447 * Iterates over the list of the enabled composites calling specified function 448 * @param task 449 * @param cache 450 * @param func 451 * @param fd 452 */ 453 void rspamd_symcache_composites_foreach (struct rspamd_task *task, 454 struct rspamd_symcache *cache, 455 GHFunc func, 456 gpointer fd); 457 458 /** 459 * Sets allowed settings ids for a symbol 460 * @param cache 461 * @param symbol 462 * @param ids 463 * @param nids 464 */ 465 bool rspamd_symcache_set_allowed_settings_ids (struct rspamd_symcache *cache, 466 const gchar *symbol, 467 const guint32 *ids, 468 guint nids); 469 /** 470 * Sets denied settings ids for a symbol 471 * @param cache 472 * @param symbol 473 * @param ids 474 * @param nids 475 */ 476 bool rspamd_symcache_set_forbidden_settings_ids (struct rspamd_symcache *cache, 477 const gchar *symbol, 478 const guint32 *ids, 479 guint nids); 480 481 /** 482 * Returns allowed ids for a symbol as a constant array 483 * @param cache 484 * @param symbol 485 * @param nids 486 * @return 487 */ 488 const guint32 *rspamd_symcache_get_allowed_settings_ids (struct rspamd_symcache *cache, 489 const gchar *symbol, 490 guint *nids); 491 492 /** 493 * Returns denied ids for a symbol as a constant array 494 * @param cache 495 * @param symbol 496 * @param nids 497 * @return 498 */ 499 const guint32 *rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache, 500 const gchar *symbol, 501 guint *nids); 502 503 504 /** 505 * Processes settings_elt in cache and converts it to a set of 506 * adjustments for forbidden/allowed settings_ids for each symbol 507 * @param cache 508 * @param elt 509 */ 510 void rspamd_symcache_process_settings_elt (struct rspamd_symcache *cache, 511 struct rspamd_config_settings_elt *elt); 512 513 /** 514 * Check if a symbol is allowed for execution/insertion, this does not involve 515 * condition scripts to be checked (so it is intended to be fast). 516 * @param task 517 * @param item 518 * @param exec_only 519 * @return 520 */ 521 gboolean rspamd_symcache_is_item_allowed (struct rspamd_task *task, 522 struct rspamd_symcache_item *item, 523 gboolean exec_only); 524 525 /** 526 * Returns symbcache item flags 527 * @param item 528 * @return 529 */ 530 gint rspamd_symcache_item_flags (struct rspamd_symcache_item *item); 531 /** 532 * Returns cache item name 533 * @param item 534 * @return 535 */ 536 const gchar* rspamd_symcache_item_name (struct rspamd_symcache_item *item); 537 /** 538 * Returns the current item stat 539 * @param item 540 * @return 541 */ 542 const struct rspamd_symcache_item_stat * 543 rspamd_symcache_item_stat (struct rspamd_symcache_item *item); 544 /** 545 * Returns if an item is enabled (for virutal it also means that parent should be enabled) 546 * @param item 547 * @return 548 */ 549 gboolean rspamd_symcache_item_is_enabled (struct rspamd_symcache_item *item); 550 /** 551 * Returns parent for virtual symbols (or NULL) 552 * @param item 553 * @return 554 */ 555 struct rspamd_symcache_item * rspamd_symcache_item_get_parent ( 556 struct rspamd_symcache_item *item); 557 /** 558 * Returns direct deps for an element 559 * @param item 560 * @return array of struct rspamd_symcache_item * 561 */ 562 const GPtrArray* rspamd_symcache_item_get_deps ( 563 struct rspamd_symcache_item *item); 564 /** 565 * Returns direct reverse deps for an element 566 * @param item 567 * @return array of struct rspamd_symcache_item * 568 */ 569 const GPtrArray* rspamd_symcache_item_get_rdeps ( 570 struct rspamd_symcache_item *item); 571 572 573 /** 574 * Enable profiling for task (e.g. when a slow rule has been found) 575 * @param task 576 */ 577 void rspamd_symcache_enable_profile (struct rspamd_task *task); 578 #ifdef __cplusplus 579 } 580 #endif 581 582 #endif 583