1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <linux/hash.h> 5 #include <linux/hashtable.h> 6 #include <linux/jhash.h> 7 #include <linux/math64.h> 8 #include <linux/vmalloc.h> 9 #include <net/pkt_cls.h> 10 11 #include "cmsg.h" 12 #include "conntrack.h" 13 #include "main.h" 14 #include "../nfp_app.h" 15 16 struct nfp_mask_id_table { 17 struct hlist_node link; 18 u32 hash_key; 19 u32 ref_cnt; 20 u8 mask_id; 21 }; 22 23 struct nfp_fl_flow_table_cmp_arg { 24 struct net_device *netdev; 25 unsigned long cookie; 26 }; 27 28 struct nfp_fl_stats_ctx_to_flow { 29 struct rhash_head ht_node; 30 u32 stats_cxt; 31 struct nfp_fl_payload *flow; 32 }; 33 34 static const struct rhashtable_params stats_ctx_table_params = { 35 .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), 36 .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), 37 .key_len = sizeof(u32), 38 }; 39 40 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) 41 { 42 struct nfp_flower_priv *priv = app->priv; 43 struct circ_buf *ring; 44 45 ring = &priv->stats_ids.free_list; 46 /* Check if buffer is full, stats_ring_size must be power of 2 */ 47 if (!CIRC_SPACE(ring->head, ring->tail, priv->stats_ring_size)) 48 return -ENOBUFS; 49 50 /* Each increment of head represents size of NFP_FL_STATS_ELEM_RS */ 51 memcpy(&ring->buf[ring->head * NFP_FL_STATS_ELEM_RS], 52 &stats_context_id, NFP_FL_STATS_ELEM_RS); 53 ring->head = (ring->head + 1) & (priv->stats_ring_size - 1); 54 55 return 0; 56 } 57 58 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) 59 { 60 struct nfp_flower_priv *priv = app->priv; 61 u32 freed_stats_id, temp_stats_id; 62 struct circ_buf *ring; 63 64 ring = &priv->stats_ids.free_list; 65 freed_stats_id = priv->stats_ring_size; 66 /* Check for unallocated entries first. */ 67 if (priv->stats_ids.init_unalloc > 0) { 68 *stats_context_id = 69 FIELD_PREP(NFP_FL_STAT_ID_STAT, 70 priv->stats_ids.init_unalloc - 1) | 71 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, 72 priv->active_mem_unit); 73 74 if (++priv->active_mem_unit == priv->total_mem_units) { 75 priv->stats_ids.init_unalloc--; 76 priv->active_mem_unit = 0; 77 } 78 79 return 0; 80 } 81 82 /* Check if buffer is empty. */ 83 if (ring->head == ring->tail) { 84 *stats_context_id = freed_stats_id; 85 return -ENOENT; 86 } 87 88 /* Each increment of tail represents size of NFP_FL_STATS_ELEM_RS */ 89 memcpy(&temp_stats_id, &ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], 90 NFP_FL_STATS_ELEM_RS); 91 *stats_context_id = temp_stats_id; 92 memcpy(&ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], &freed_stats_id, 93 NFP_FL_STATS_ELEM_RS); 94 /* stats_ring_size must be power of 2 */ 95 ring->tail = (ring->tail + 1) & (priv->stats_ring_size - 1); 96 97 return 0; 98 } 99 100 /* Must be called with either RTNL or rcu_read_lock */ 101 struct nfp_fl_payload * 102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, 103 struct net_device *netdev) 104 { 105 struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; 106 struct nfp_flower_priv *priv = app->priv; 107 108 flower_cmp_arg.netdev = netdev; 109 flower_cmp_arg.cookie = tc_flower_cookie; 110 111 return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, 112 nfp_flower_table_params); 113 } 114 115 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) 116 { 117 unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); 118 struct nfp_flower_priv *priv = app->priv; 119 struct nfp_fl_stats_frame *stats; 120 unsigned char *msg; 121 u32 ctx_id; 122 int i; 123 124 msg = nfp_flower_cmsg_get_data(skb); 125 126 spin_lock(&priv->stats_lock); 127 for (i = 0; i < msg_len / sizeof(*stats); i++) { 128 stats = (struct nfp_fl_stats_frame *)msg + i; 129 ctx_id = be32_to_cpu(stats->stats_con_id); 130 priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count); 131 priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count); 132 priv->stats[ctx_id].used = jiffies; 133 } 134 spin_unlock(&priv->stats_lock); 135 } 136 137 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) 138 { 139 struct nfp_flower_priv *priv = app->priv; 140 struct circ_buf *ring; 141 142 ring = &priv->mask_ids.mask_id_free_list; 143 /* Checking if buffer is full, 144 * NFP_FLOWER_MASK_ENTRY_RS must be power of 2 145 */ 146 if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) 147 return -ENOBUFS; 148 149 /* Each increment of head represents size of 150 * NFP_FLOWER_MASK_ELEMENT_RS 151 */ 152 memcpy(&ring->buf[ring->head * NFP_FLOWER_MASK_ELEMENT_RS], &mask_id, 153 NFP_FLOWER_MASK_ELEMENT_RS); 154 ring->head = (ring->head + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); 155 156 priv->mask_ids.last_used[mask_id] = ktime_get(); 157 158 return 0; 159 } 160 161 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) 162 { 163 struct nfp_flower_priv *priv = app->priv; 164 ktime_t reuse_timeout; 165 struct circ_buf *ring; 166 u8 temp_id, freed_id; 167 168 ring = &priv->mask_ids.mask_id_free_list; 169 freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; 170 /* Checking for unallocated entries first. */ 171 if (priv->mask_ids.init_unallocated > 0) { 172 *mask_id = priv->mask_ids.init_unallocated; 173 priv->mask_ids.init_unallocated--; 174 return 0; 175 } 176 177 /* Checking if buffer is empty. */ 178 if (ring->head == ring->tail) 179 goto err_not_found; 180 181 /* Each increment of tail represents size of 182 * NFP_FLOWER_MASK_ELEMENT_RS 183 */ 184 memcpy(&temp_id, &ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], 185 NFP_FLOWER_MASK_ELEMENT_RS); 186 *mask_id = temp_id; 187 188 reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id], 189 NFP_FL_MASK_REUSE_TIME_NS); 190 191 if (ktime_before(ktime_get(), reuse_timeout)) 192 goto err_not_found; 193 194 memcpy(&ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], &freed_id, 195 NFP_FLOWER_MASK_ELEMENT_RS); 196 /* NFP_FLOWER_MASK_ENTRY_RS must be power of 2 */ 197 ring->tail = (ring->tail + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); 198 199 return 0; 200 201 err_not_found: 202 *mask_id = freed_id; 203 return -ENOENT; 204 } 205 206 static int 207 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 208 { 209 struct nfp_flower_priv *priv = app->priv; 210 struct nfp_mask_id_table *mask_entry; 211 unsigned long hash_key; 212 u8 mask_id; 213 214 if (nfp_mask_alloc(app, &mask_id)) 215 return -ENOENT; 216 217 mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL); 218 if (!mask_entry) { 219 nfp_release_mask_id(app, mask_id); 220 return -ENOMEM; 221 } 222 223 INIT_HLIST_NODE(&mask_entry->link); 224 mask_entry->mask_id = mask_id; 225 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 226 mask_entry->hash_key = hash_key; 227 mask_entry->ref_cnt = 1; 228 hash_add(priv->mask_table, &mask_entry->link, hash_key); 229 230 return mask_id; 231 } 232 233 static struct nfp_mask_id_table * 234 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 235 { 236 struct nfp_flower_priv *priv = app->priv; 237 struct nfp_mask_id_table *mask_entry; 238 unsigned long hash_key; 239 240 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 241 242 hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) 243 if (mask_entry->hash_key == hash_key) 244 return mask_entry; 245 246 return NULL; 247 } 248 249 static int 250 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 251 { 252 struct nfp_mask_id_table *mask_entry; 253 254 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 255 if (!mask_entry) 256 return -ENOENT; 257 258 mask_entry->ref_cnt++; 259 260 /* Casting u8 to int for later use. */ 261 return mask_entry->mask_id; 262 } 263 264 static bool 265 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, 266 u8 *meta_flags, u8 *mask_id) 267 { 268 int id; 269 270 id = nfp_find_in_mask_table(app, mask_data, mask_len); 271 if (id < 0) { 272 id = nfp_add_mask_table(app, mask_data, mask_len); 273 if (id < 0) 274 return false; 275 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 276 } 277 *mask_id = id; 278 279 return true; 280 } 281 282 static bool 283 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, 284 u8 *meta_flags, u8 *mask_id) 285 { 286 struct nfp_mask_id_table *mask_entry; 287 288 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 289 if (!mask_entry) 290 return false; 291 292 *mask_id = mask_entry->mask_id; 293 mask_entry->ref_cnt--; 294 if (!mask_entry->ref_cnt) { 295 hash_del(&mask_entry->link); 296 nfp_release_mask_id(app, *mask_id); 297 kfree(mask_entry); 298 if (meta_flags) 299 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 300 } 301 302 return true; 303 } 304 305 int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie, 306 struct nfp_fl_payload *nfp_flow, 307 struct net_device *netdev, 308 struct netlink_ext_ack *extack) 309 { 310 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 311 struct nfp_flower_priv *priv = app->priv; 312 struct nfp_fl_payload *check_entry; 313 u8 new_mask_id; 314 u32 stats_cxt; 315 int err; 316 317 err = nfp_get_stats_entry(app, &stats_cxt); 318 if (err) { 319 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context"); 320 return err; 321 } 322 323 nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); 324 nfp_flow->meta.host_cookie = cpu_to_be64(cookie); 325 nfp_flow->ingress_dev = netdev; 326 327 ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL); 328 if (!ctx_entry) { 329 err = -ENOMEM; 330 goto err_release_stats; 331 } 332 333 ctx_entry->stats_cxt = stats_cxt; 334 ctx_entry->flow = nfp_flow; 335 336 if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node, 337 stats_ctx_table_params)) { 338 err = -ENOMEM; 339 goto err_free_ctx_entry; 340 } 341 342 /* Do net allocate a mask-id for pre_tun_rules. These flows are used to 343 * configure the pre_tun table and are never actually send to the 344 * firmware as an add-flow message. This causes the mask-id allocation 345 * on the firmware to get out of sync if allocated here. 346 */ 347 new_mask_id = 0; 348 if (!nfp_flow->pre_tun_rule.dev && 349 !nfp_check_mask_add(app, nfp_flow->mask_data, 350 nfp_flow->meta.mask_len, 351 &nfp_flow->meta.flags, &new_mask_id)) { 352 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id"); 353 err = -ENOENT; 354 goto err_remove_rhash; 355 } 356 357 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 358 priv->flower_version++; 359 360 /* Update flow payload with mask ids. */ 361 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 362 priv->stats[stats_cxt].pkts = 0; 363 priv->stats[stats_cxt].bytes = 0; 364 priv->stats[stats_cxt].used = jiffies; 365 366 check_entry = nfp_flower_search_fl_table(app, cookie, netdev); 367 if (check_entry) { 368 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry"); 369 err = -EEXIST; 370 goto err_remove_mask; 371 } 372 373 return 0; 374 375 err_remove_mask: 376 if (!nfp_flow->pre_tun_rule.dev) 377 nfp_check_mask_remove(app, nfp_flow->mask_data, 378 nfp_flow->meta.mask_len, 379 NULL, &new_mask_id); 380 err_remove_rhash: 381 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 382 &ctx_entry->ht_node, 383 stats_ctx_table_params)); 384 err_free_ctx_entry: 385 kfree(ctx_entry); 386 err_release_stats: 387 nfp_release_stats_entry(app, stats_cxt); 388 389 return err; 390 } 391 392 void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, 393 struct nfp_fl_payload *nfp_flow) 394 { 395 nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; 396 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 397 priv->flower_version++; 398 } 399 400 int nfp_modify_flow_metadata(struct nfp_app *app, 401 struct nfp_fl_payload *nfp_flow) 402 { 403 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 404 struct nfp_flower_priv *priv = app->priv; 405 u8 new_mask_id = 0; 406 u32 temp_ctx_id; 407 408 __nfp_modify_flow_metadata(priv, nfp_flow); 409 410 if (!nfp_flow->pre_tun_rule.dev) 411 nfp_check_mask_remove(app, nfp_flow->mask_data, 412 nfp_flow->meta.mask_len, &nfp_flow->meta.flags, 413 &new_mask_id); 414 415 /* Update flow payload with mask ids. */ 416 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 417 418 /* Release the stats ctx id and ctx to flow table entry. */ 419 temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); 420 421 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id, 422 stats_ctx_table_params); 423 if (!ctx_entry) 424 return -ENOENT; 425 426 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 427 &ctx_entry->ht_node, 428 stats_ctx_table_params)); 429 kfree(ctx_entry); 430 431 return nfp_release_stats_entry(app, temp_ctx_id); 432 } 433 434 struct nfp_fl_payload * 435 nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) 436 { 437 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 438 struct nfp_flower_priv *priv = app->priv; 439 440 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id, 441 stats_ctx_table_params); 442 if (!ctx_entry) 443 return NULL; 444 445 return ctx_entry->flow; 446 } 447 448 static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, 449 const void *obj) 450 { 451 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; 452 const struct nfp_fl_payload *flow_entry = obj; 453 454 if (flow_entry->ingress_dev == cmp_arg->netdev) 455 return flow_entry->tc_flower_cookie != cmp_arg->cookie; 456 457 return 1; 458 } 459 460 static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) 461 { 462 const struct nfp_fl_payload *flower_entry = data; 463 464 return jhash2((u32 *)&flower_entry->tc_flower_cookie, 465 sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), 466 seed); 467 } 468 469 static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) 470 { 471 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; 472 473 return jhash2((u32 *)&cmp_arg->cookie, 474 sizeof(cmp_arg->cookie) / sizeof(u32), seed); 475 } 476 477 const struct rhashtable_params nfp_flower_table_params = { 478 .head_offset = offsetof(struct nfp_fl_payload, fl_node), 479 .hashfn = nfp_fl_key_hashfn, 480 .obj_cmpfn = nfp_fl_obj_cmpfn, 481 .obj_hashfn = nfp_fl_obj_hashfn, 482 .automatic_shrinking = true, 483 }; 484 485 const struct rhashtable_params merge_table_params = { 486 .key_offset = offsetof(struct nfp_merge_info, parent_ctx), 487 .head_offset = offsetof(struct nfp_merge_info, ht_node), 488 .key_len = sizeof(u64), 489 }; 490 491 const struct rhashtable_params nfp_zone_table_params = { 492 .head_offset = offsetof(struct nfp_fl_ct_zone_entry, hash_node), 493 .key_len = sizeof(u16), 494 .key_offset = offsetof(struct nfp_fl_ct_zone_entry, zone), 495 .automatic_shrinking = false, 496 }; 497 498 const struct rhashtable_params nfp_ct_map_params = { 499 .head_offset = offsetof(struct nfp_fl_ct_map_entry, hash_node), 500 .key_len = sizeof(unsigned long), 501 .key_offset = offsetof(struct nfp_fl_ct_map_entry, cookie), 502 .automatic_shrinking = true, 503 }; 504 505 int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, 506 unsigned int host_num_mems) 507 { 508 struct nfp_flower_priv *priv = app->priv; 509 int err, stats_size; 510 511 hash_init(priv->mask_table); 512 513 err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params); 514 if (err) 515 return err; 516 517 err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params); 518 if (err) 519 goto err_free_flow_table; 520 521 err = rhashtable_init(&priv->merge_table, &merge_table_params); 522 if (err) 523 goto err_free_stats_ctx_table; 524 525 err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params); 526 if (err) 527 goto err_free_merge_table; 528 529 err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params); 530 if (err) 531 goto err_free_ct_zone_table; 532 533 get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); 534 535 /* Init ring buffer and unallocated mask_ids. */ 536 priv->mask_ids.mask_id_free_list.buf = 537 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 538 NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); 539 if (!priv->mask_ids.mask_id_free_list.buf) 540 goto err_free_ct_map_table; 541 542 priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; 543 544 /* Init timestamps for mask id*/ 545 priv->mask_ids.last_used = 546 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 547 sizeof(*priv->mask_ids.last_used), GFP_KERNEL); 548 if (!priv->mask_ids.last_used) 549 goto err_free_mask_id; 550 551 /* Init ring buffer and unallocated stats_ids. */ 552 priv->stats_ids.free_list.buf = 553 vmalloc(array_size(NFP_FL_STATS_ELEM_RS, 554 priv->stats_ring_size)); 555 if (!priv->stats_ids.free_list.buf) 556 goto err_free_last_used; 557 558 priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems); 559 560 stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) | 561 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1); 562 priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats), 563 GFP_KERNEL); 564 if (!priv->stats) 565 goto err_free_ring_buf; 566 567 spin_lock_init(&priv->stats_lock); 568 569 return 0; 570 571 err_free_ring_buf: 572 vfree(priv->stats_ids.free_list.buf); 573 err_free_last_used: 574 kfree(priv->mask_ids.last_used); 575 err_free_mask_id: 576 kfree(priv->mask_ids.mask_id_free_list.buf); 577 err_free_ct_map_table: 578 rhashtable_destroy(&priv->ct_map_table); 579 err_free_ct_zone_table: 580 rhashtable_destroy(&priv->ct_zone_table); 581 err_free_merge_table: 582 rhashtable_destroy(&priv->merge_table); 583 err_free_stats_ctx_table: 584 rhashtable_destroy(&priv->stats_ctx_table); 585 err_free_flow_table: 586 rhashtable_destroy(&priv->flow_table); 587 return -ENOMEM; 588 } 589 590 static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt) 591 { 592 if (!zt) 593 return; 594 595 if (!list_empty(&zt->pre_ct_list)) { 596 struct rhashtable *m_table = &zt->priv->ct_map_table; 597 struct nfp_fl_ct_flow_entry *entry, *tmp; 598 struct nfp_fl_ct_map_entry *map; 599 600 WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n"); 601 list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list, 602 list_node) { 603 map = rhashtable_lookup_fast(m_table, 604 &entry->cookie, 605 nfp_ct_map_params); 606 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 607 &map->hash_node, 608 nfp_ct_map_params)); 609 nfp_fl_ct_clean_flow_entry(entry); 610 kfree(map); 611 } 612 } 613 614 if (!list_empty(&zt->post_ct_list)) { 615 struct rhashtable *m_table = &zt->priv->ct_map_table; 616 struct nfp_fl_ct_flow_entry *entry, *tmp; 617 struct nfp_fl_ct_map_entry *map; 618 619 WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n"); 620 list_for_each_entry_safe(entry, tmp, &zt->post_ct_list, 621 list_node) { 622 map = rhashtable_lookup_fast(m_table, 623 &entry->cookie, 624 nfp_ct_map_params); 625 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 626 &map->hash_node, 627 nfp_ct_map_params)); 628 nfp_fl_ct_clean_flow_entry(entry); 629 kfree(map); 630 } 631 } 632 633 if (zt->nft) { 634 nf_flow_table_offload_del_cb(zt->nft, 635 nfp_fl_ct_handle_nft_flow, 636 zt); 637 zt->nft = NULL; 638 } 639 640 if (!list_empty(&zt->nft_flows_list)) { 641 struct rhashtable *m_table = &zt->priv->ct_map_table; 642 struct nfp_fl_ct_flow_entry *entry, *tmp; 643 struct nfp_fl_ct_map_entry *map; 644 645 WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n"); 646 list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list, 647 list_node) { 648 map = rhashtable_lookup_fast(m_table, 649 &entry->cookie, 650 nfp_ct_map_params); 651 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 652 &map->hash_node, 653 nfp_ct_map_params)); 654 nfp_fl_ct_clean_flow_entry(entry); 655 kfree(map); 656 } 657 } 658 659 rhashtable_free_and_destroy(&zt->tc_merge_tb, 660 nfp_check_rhashtable_empty, NULL); 661 rhashtable_free_and_destroy(&zt->nft_merge_tb, 662 nfp_check_rhashtable_empty, NULL); 663 664 kfree(zt); 665 } 666 667 static void nfp_free_zone_table_entry(void *ptr, void *arg) 668 { 669 struct nfp_fl_ct_zone_entry *zt = ptr; 670 671 nfp_zone_table_entry_destroy(zt); 672 } 673 674 static void nfp_free_map_table_entry(void *ptr, void *arg) 675 { 676 struct nfp_fl_ct_map_entry *map = ptr; 677 678 if (!map) 679 return; 680 681 kfree(map); 682 } 683 684 void nfp_flower_metadata_cleanup(struct nfp_app *app) 685 { 686 struct nfp_flower_priv *priv = app->priv; 687 688 if (!priv) 689 return; 690 691 rhashtable_free_and_destroy(&priv->flow_table, 692 nfp_check_rhashtable_empty, NULL); 693 rhashtable_free_and_destroy(&priv->stats_ctx_table, 694 nfp_check_rhashtable_empty, NULL); 695 rhashtable_free_and_destroy(&priv->merge_table, 696 nfp_check_rhashtable_empty, NULL); 697 rhashtable_free_and_destroy(&priv->ct_zone_table, 698 nfp_free_zone_table_entry, NULL); 699 nfp_zone_table_entry_destroy(priv->ct_zone_wc); 700 701 rhashtable_free_and_destroy(&priv->ct_map_table, 702 nfp_free_map_table_entry, NULL); 703 kvfree(priv->stats); 704 kfree(priv->mask_ids.mask_id_free_list.buf); 705 kfree(priv->mask_ids.last_used); 706 vfree(priv->stats_ids.free_list.buf); 707 } 708