1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/bitmap.h> 5 #include <linux/errno.h> 6 #include <linux/genalloc.h> 7 #include <linux/gfp.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/mutex.h> 11 #include <linux/objagg.h> 12 #include <linux/rtnetlink.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "reg.h" 17 #include "spectrum.h" 18 #include "spectrum_acl_tcam.h" 19 20 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ 21 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100 22 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16 23 24 struct mlxsw_sp_acl_erp_core { 25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1]; 26 struct gen_pool *erp_tables; 27 struct mlxsw_sp *mlxsw_sp; 28 struct mlxsw_sp_acl_bf *bf; 29 unsigned int num_erp_banks; 30 }; 31 32 struct mlxsw_sp_acl_erp_key { 33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; 34 #define __MASK_LEN 0x38 35 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1) 36 bool ctcam; 37 }; 38 39 struct mlxsw_sp_acl_erp { 40 struct mlxsw_sp_acl_erp_key key; 41 u8 id; 42 u8 index; 43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 44 struct list_head list; 45 struct mlxsw_sp_acl_erp_table *erp_table; 46 }; 47 48 struct mlxsw_sp_acl_erp_master_mask { 49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN]; 51 }; 52 53 struct mlxsw_sp_acl_erp_table { 54 struct mlxsw_sp_acl_erp_master_mask master_mask; 55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 57 struct list_head atcam_erps_list; 58 struct mlxsw_sp_acl_erp_core *erp_core; 59 struct mlxsw_sp_acl_atcam_region *aregion; 60 const struct mlxsw_sp_acl_erp_table_ops *ops; 61 unsigned long base_index; 62 unsigned int num_atcam_erps; 63 unsigned int num_max_atcam_erps; 64 unsigned int num_ctcam_erps; 65 unsigned int num_deltas; 66 struct objagg *objagg; 67 struct mutex objagg_lock; /* guards objagg manipulation */ 68 }; 69 70 struct mlxsw_sp_acl_erp_table_ops { 71 struct mlxsw_sp_acl_erp * 72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table, 73 struct mlxsw_sp_acl_erp_key *key); 74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table, 75 struct mlxsw_sp_acl_erp *erp); 76 }; 77 78 static struct mlxsw_sp_acl_erp * 79 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 80 struct mlxsw_sp_acl_erp_key *key); 81 static void 82 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 83 struct mlxsw_sp_acl_erp *erp); 84 static struct mlxsw_sp_acl_erp * 85 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 86 struct mlxsw_sp_acl_erp_key *key); 87 static void 88 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 89 struct mlxsw_sp_acl_erp *erp); 90 static struct mlxsw_sp_acl_erp * 91 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 92 struct mlxsw_sp_acl_erp_key *key); 93 static void 94 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 95 struct mlxsw_sp_acl_erp *erp); 96 static void 97 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 98 struct mlxsw_sp_acl_erp *erp); 99 100 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = { 101 .erp_create = mlxsw_sp_acl_erp_mask_create, 102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy, 103 }; 104 105 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = { 106 .erp_create = mlxsw_sp_acl_erp_mask_create, 107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy, 108 }; 109 110 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = { 111 .erp_create = mlxsw_sp_acl_erp_second_mask_create, 112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy, 113 }; 114 115 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = { 116 .erp_create = mlxsw_sp_acl_erp_first_mask_create, 117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy, 118 }; 119 120 static bool 121 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table) 122 { 123 return erp_table->ops != &erp_single_mask_ops && 124 erp_table->ops != &erp_no_mask_ops; 125 } 126 127 static unsigned int 128 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp) 129 { 130 return erp->index % erp->erp_table->erp_core->num_erp_banks; 131 } 132 133 static unsigned int 134 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table) 135 { 136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 138 139 return erp_core->erpt_entries_size[aregion->type]; 140 } 141 142 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table, 143 u8 *p_id) 144 { 145 u8 id; 146 147 id = find_first_zero_bit(erp_table->erp_id_bitmap, 148 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) { 150 __set_bit(id, erp_table->erp_id_bitmap); 151 *p_id = id; 152 return 0; 153 } 154 155 return -ENOBUFS; 156 } 157 158 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table, 159 u8 id) 160 { 161 __clear_bit(id, erp_table->erp_id_bitmap); 162 } 163 164 static void 165 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit, 166 struct mlxsw_sp_acl_erp_master_mask *mask) 167 { 168 if (mask->count[bit]++ == 0) 169 __set_bit(bit, mask->bitmap); 170 } 171 172 static void 173 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit, 174 struct mlxsw_sp_acl_erp_master_mask *mask) 175 { 176 if (--mask->count[bit] == 0) 177 __clear_bit(bit, mask->bitmap); 178 } 179 180 static int 181 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table) 182 { 183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 185 char percr_pl[MLXSW_REG_PERCR_LEN]; 186 char *master_mask; 187 188 mlxsw_reg_percr_pack(percr_pl, region->id); 189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl); 190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap, 191 MLXSW_SP_ACL_TCAM_MASK_LEN); 192 193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 194 } 195 196 static int 197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table, 198 struct mlxsw_sp_acl_erp_key *key) 199 { 200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 201 unsigned long bit; 202 int err; 203 204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 205 MLXSW_SP_ACL_TCAM_MASK_LEN); 206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 207 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 208 &erp_table->master_mask); 209 210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 211 if (err) 212 goto err_master_mask_update; 213 214 return 0; 215 216 err_master_mask_update: 217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 219 &erp_table->master_mask); 220 return err; 221 } 222 223 static int 224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table, 225 struct mlxsw_sp_acl_erp_key *key) 226 { 227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 228 unsigned long bit; 229 int err; 230 231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 232 MLXSW_SP_ACL_TCAM_MASK_LEN); 233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 235 &erp_table->master_mask); 236 237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 238 if (err) 239 goto err_master_mask_update; 240 241 return 0; 242 243 err_master_mask_update: 244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 245 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 246 &erp_table->master_mask); 247 return err; 248 } 249 250 static struct mlxsw_sp_acl_erp * 251 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table, 252 struct mlxsw_sp_acl_erp_key *key) 253 { 254 struct mlxsw_sp_acl_erp *erp; 255 int err; 256 257 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 258 if (!erp) 259 return ERR_PTR(-ENOMEM); 260 261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id); 262 if (err) 263 goto err_erp_id_get; 264 265 memcpy(&erp->key, key, sizeof(*key)); 266 list_add(&erp->list, &erp_table->atcam_erps_list); 267 erp_table->num_atcam_erps++; 268 erp->erp_table = erp_table; 269 270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 271 if (err) 272 goto err_master_mask_set; 273 274 return erp; 275 276 err_master_mask_set: 277 erp_table->num_atcam_erps--; 278 list_del(&erp->list); 279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 280 err_erp_id_get: 281 kfree(erp); 282 return ERR_PTR(err); 283 } 284 285 static void 286 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp) 287 { 288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 289 290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 291 erp_table->num_atcam_erps--; 292 list_del(&erp->list); 293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 294 kfree(erp); 295 } 296 297 static int 298 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, 299 unsigned int num_erps, 300 enum mlxsw_sp_acl_atcam_region_type region_type, 301 unsigned long *p_index) 302 { 303 unsigned int num_rows, entry_size; 304 305 /* We only allow allocations of entire rows */ 306 if (num_erps % erp_core->num_erp_banks != 0) 307 return -EINVAL; 308 309 entry_size = erp_core->erpt_entries_size[region_type]; 310 num_rows = num_erps / erp_core->num_erp_banks; 311 312 *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); 313 if (*p_index == 0) 314 return -ENOBUFS; 315 *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 316 317 return 0; 318 } 319 320 static void 321 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core, 322 unsigned int num_erps, 323 enum mlxsw_sp_acl_atcam_region_type region_type, 324 unsigned long index) 325 { 326 unsigned long base_index; 327 unsigned int entry_size; 328 size_t size; 329 330 entry_size = erp_core->erpt_entries_size[region_type]; 331 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 332 size = num_erps / erp_core->num_erp_banks * entry_size; 333 gen_pool_free(erp_core->erp_tables, base_index, size); 334 } 335 336 static struct mlxsw_sp_acl_erp * 337 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table) 338 { 339 if (!list_is_singular(&erp_table->atcam_erps_list)) 340 return NULL; 341 342 return list_first_entry(&erp_table->atcam_erps_list, 343 struct mlxsw_sp_acl_erp, list); 344 } 345 346 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table, 347 u8 *p_index) 348 { 349 u8 index; 350 351 index = find_first_zero_bit(erp_table->erp_index_bitmap, 352 erp_table->num_max_atcam_erps); 353 if (index < erp_table->num_max_atcam_erps) { 354 __set_bit(index, erp_table->erp_index_bitmap); 355 *p_index = index; 356 return 0; 357 } 358 359 return -ENOBUFS; 360 } 361 362 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table, 363 u8 index) 364 { 365 __clear_bit(index, erp_table->erp_index_bitmap); 366 } 367 368 static void 369 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table, 370 const struct mlxsw_sp_acl_erp *erp, 371 u8 *p_erpt_bank, u8 *p_erpt_index) 372 { 373 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table); 374 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 375 unsigned int row; 376 377 *p_erpt_bank = erp->index % erp_core->num_erp_banks; 378 row = erp->index / erp_core->num_erp_banks; 379 *p_erpt_index = erp_table->base_index + row * entry_size; 380 } 381 382 static int 383 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 384 struct mlxsw_sp_acl_erp *erp) 385 { 386 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 387 enum mlxsw_reg_perpt_key_size key_size; 388 char perpt_pl[MLXSW_REG_PERPT_LEN]; 389 u8 erpt_bank, erpt_index; 390 391 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 392 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 393 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 394 0, erp_table->base_index, erp->index, 395 erp->key.mask); 396 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 397 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 398 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true); 399 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 400 } 401 402 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp) 403 { 404 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 405 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 406 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 407 enum mlxsw_reg_perpt_key_size key_size; 408 char perpt_pl[MLXSW_REG_PERPT_LEN]; 409 u8 erpt_bank, erpt_index; 410 411 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 412 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 413 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 414 0, erp_table->base_index, erp->index, empty_mask); 415 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 416 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 417 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false); 418 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 419 } 420 421 static int 422 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table, 423 bool ctcam_le) 424 { 425 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 426 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 427 char pererp_pl[MLXSW_REG_PERERP_LEN]; 428 429 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 430 erp_table->base_index, 0); 431 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 432 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 433 434 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 435 } 436 437 static void 438 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table) 439 { 440 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 441 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 442 char pererp_pl[MLXSW_REG_PERERP_LEN]; 443 struct mlxsw_sp_acl_erp *master_rp; 444 445 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 446 /* It is possible we do not have a master RP when we disable the 447 * table when there are no rules in the A-TCAM and the last C-TCAM 448 * rule is deleted 449 */ 450 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0, 451 master_rp ? master_rp->id : 0); 452 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 453 } 454 455 static int 456 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table) 457 { 458 struct mlxsw_sp_acl_erp *erp; 459 int err; 460 461 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) { 462 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 463 if (err) 464 goto err_table_erp_add; 465 } 466 467 return 0; 468 469 err_table_erp_add: 470 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list, 471 list) 472 mlxsw_sp_acl_erp_table_erp_del(erp); 473 return err; 474 } 475 476 static int 477 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table) 478 { 479 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps; 480 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 481 unsigned long old_base_index = erp_table->base_index; 482 bool ctcam_le = erp_table->num_ctcam_erps > 0; 483 int err; 484 485 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps) 486 return 0; 487 488 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION) 489 return -ENOBUFS; 490 491 num_erps = old_num_erps + erp_core->num_erp_banks; 492 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps, 493 erp_table->aregion->type, 494 &erp_table->base_index); 495 if (err) 496 return err; 497 erp_table->num_max_atcam_erps = num_erps; 498 499 err = mlxsw_sp_acl_erp_table_relocate(erp_table); 500 if (err) 501 goto err_table_relocate; 502 503 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le); 504 if (err) 505 goto err_table_enable; 506 507 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps, 508 erp_table->aregion->type, old_base_index); 509 510 return 0; 511 512 err_table_enable: 513 err_table_relocate: 514 erp_table->num_max_atcam_erps = old_num_erps; 515 mlxsw_sp_acl_erp_table_free(erp_core, num_erps, 516 erp_table->aregion->type, 517 erp_table->base_index); 518 erp_table->base_index = old_base_index; 519 return err; 520 } 521 522 static int 523 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table, 524 struct mlxsw_sp_acl_erp *erp) 525 { 526 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 527 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 528 struct mlxsw_sp_acl_atcam_entry *aentry; 529 int err; 530 531 list_for_each_entry(aentry, &aregion->entries_list, list) { 532 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp, 533 erp_table->erp_core->bf, 534 aregion, erp_bank, aentry); 535 if (err) 536 goto bf_entry_add_err; 537 } 538 539 return 0; 540 541 bf_entry_add_err: 542 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list, 543 list) 544 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 545 erp_table->erp_core->bf, 546 aregion, erp_bank, aentry); 547 return err; 548 } 549 550 static void 551 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table, 552 struct mlxsw_sp_acl_erp *erp) 553 { 554 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 555 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 556 struct mlxsw_sp_acl_atcam_entry *aentry; 557 558 list_for_each_entry_reverse(aentry, &aregion->entries_list, list) 559 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 560 erp_table->erp_core->bf, 561 aregion, erp_bank, aentry); 562 } 563 564 static int 565 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table) 566 { 567 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 568 struct mlxsw_sp_acl_erp *master_rp; 569 int err; 570 571 /* Initially, allocate a single eRP row. Expand later as needed */ 572 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks, 573 erp_table->aregion->type, 574 &erp_table->base_index); 575 if (err) 576 return err; 577 erp_table->num_max_atcam_erps = erp_core->num_erp_banks; 578 579 /* Transition the sole RP currently configured (the master RP) 580 * to the eRP table 581 */ 582 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 583 if (!master_rp) { 584 err = -EINVAL; 585 goto err_table_master_rp; 586 } 587 588 /* Make sure the master RP is using a valid index, as 589 * only a single eRP row is currently allocated. 590 */ 591 master_rp->index = 0; 592 __set_bit(master_rp->index, erp_table->erp_index_bitmap); 593 594 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp); 595 if (err) 596 goto err_table_master_rp_add; 597 598 /* Update Bloom filter before enabling eRP table, as rules 599 * on the master RP were not set to Bloom filter up to this 600 * point. 601 */ 602 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp); 603 if (err) 604 goto err_table_bf_add; 605 606 err = mlxsw_sp_acl_erp_table_enable(erp_table, false); 607 if (err) 608 goto err_table_enable; 609 610 return 0; 611 612 err_table_enable: 613 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 614 err_table_bf_add: 615 mlxsw_sp_acl_erp_table_erp_del(master_rp); 616 err_table_master_rp_add: 617 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 618 err_table_master_rp: 619 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 620 erp_table->aregion->type, 621 erp_table->base_index); 622 return err; 623 } 624 625 static void 626 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table) 627 { 628 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 629 struct mlxsw_sp_acl_erp *master_rp; 630 631 mlxsw_sp_acl_erp_table_disable(erp_table); 632 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 633 if (!master_rp) 634 return; 635 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 636 mlxsw_sp_acl_erp_table_erp_del(master_rp); 637 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 638 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 639 erp_table->aregion->type, 640 erp_table->base_index); 641 } 642 643 static int 644 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 645 struct mlxsw_sp_acl_erp *erp) 646 { 647 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 648 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 649 bool ctcam_le = erp_table->num_ctcam_erps > 0; 650 char pererp_pl[MLXSW_REG_PERERP_LEN]; 651 652 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 653 erp_table->base_index, 0); 654 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 655 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 656 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true); 657 658 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 659 } 660 661 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp) 662 { 663 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 664 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 665 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 666 bool ctcam_le = erp_table->num_ctcam_erps > 0; 667 char pererp_pl[MLXSW_REG_PERERP_LEN]; 668 669 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 670 erp_table->base_index, 0); 671 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 672 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 673 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false); 674 675 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 676 } 677 678 static int 679 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table) 680 { 681 /* No need to re-enable lookup in the C-TCAM */ 682 if (erp_table->num_ctcam_erps > 1) 683 return 0; 684 685 return mlxsw_sp_acl_erp_table_enable(erp_table, true); 686 } 687 688 static void 689 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table) 690 { 691 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */ 692 if (erp_table->num_ctcam_erps > 1) 693 return; 694 695 mlxsw_sp_acl_erp_table_enable(erp_table, false); 696 } 697 698 static int 699 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table, 700 unsigned int *inc_num) 701 { 702 int err; 703 704 /* If there are C-TCAM eRP or deltas in use we need to transition 705 * the region to use eRP table, if it is not already done 706 */ 707 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) { 708 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 709 if (err) 710 return err; 711 } 712 713 /* When C-TCAM or deltas are used, the eRP table must be used */ 714 if (erp_table->ops != &erp_multiple_masks_ops) 715 erp_table->ops = &erp_multiple_masks_ops; 716 717 (*inc_num)++; 718 719 return 0; 720 } 721 722 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table) 723 { 724 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 725 &erp_table->num_ctcam_erps); 726 } 727 728 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table) 729 { 730 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 731 &erp_table->num_deltas); 732 } 733 734 static void 735 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table, 736 unsigned int *dec_num) 737 { 738 (*dec_num)--; 739 740 /* If there are no C-TCAM eRP or deltas in use, the state we 741 * transition to depends on the number of A-TCAM eRPs currently 742 * in use. 743 */ 744 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0) 745 return; 746 747 switch (erp_table->num_atcam_erps) { 748 case 2: 749 /* Keep using the eRP table, but correctly set the 750 * operations pointer so that when an A-TCAM eRP is 751 * deleted we will transition to use the master mask 752 */ 753 erp_table->ops = &erp_two_masks_ops; 754 break; 755 case 1: 756 /* We only kept the eRP table because we had C-TCAM 757 * eRPs in use. Now that the last C-TCAM eRP is gone we 758 * can stop using the table and transition to use the 759 * master mask 760 */ 761 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 762 erp_table->ops = &erp_single_mask_ops; 763 break; 764 case 0: 765 /* There are no more eRPs of any kind used by the region 766 * so free its eRP table and transition to initial state 767 */ 768 mlxsw_sp_acl_erp_table_disable(erp_table); 769 mlxsw_sp_acl_erp_table_free(erp_table->erp_core, 770 erp_table->num_max_atcam_erps, 771 erp_table->aregion->type, 772 erp_table->base_index); 773 erp_table->ops = &erp_no_mask_ops; 774 break; 775 default: 776 break; 777 } 778 } 779 780 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table) 781 { 782 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 783 &erp_table->num_ctcam_erps); 784 } 785 786 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table) 787 { 788 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 789 &erp_table->num_deltas); 790 } 791 792 static struct mlxsw_sp_acl_erp * 793 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 794 struct mlxsw_sp_acl_erp_key *key) 795 { 796 struct mlxsw_sp_acl_erp *erp; 797 int err; 798 799 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 800 if (!erp) 801 return ERR_PTR(-ENOMEM); 802 803 memcpy(&erp->key, key, sizeof(*key)); 804 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask, 805 MLXSW_SP_ACL_TCAM_MASK_LEN); 806 807 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table); 808 if (err) 809 goto err_erp_ctcam_inc; 810 811 erp->erp_table = erp_table; 812 813 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 814 if (err) 815 goto err_master_mask_set; 816 817 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table); 818 if (err) 819 goto err_erp_region_ctcam_enable; 820 821 return erp; 822 823 err_erp_region_ctcam_enable: 824 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 825 err_master_mask_set: 826 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 827 err_erp_ctcam_inc: 828 kfree(erp); 829 return ERR_PTR(err); 830 } 831 832 static void 833 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp) 834 { 835 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 836 837 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table); 838 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 839 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 840 kfree(erp); 841 } 842 843 static struct mlxsw_sp_acl_erp * 844 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 845 struct mlxsw_sp_acl_erp_key *key) 846 { 847 struct mlxsw_sp_acl_erp *erp; 848 int err; 849 850 if (key->ctcam) 851 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 852 853 /* Expand the eRP table for the new eRP, if needed */ 854 err = mlxsw_sp_acl_erp_table_expand(erp_table); 855 if (err) 856 return ERR_PTR(err); 857 858 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 859 if (IS_ERR(erp)) 860 return erp; 861 862 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 863 if (err) 864 goto err_erp_index_get; 865 866 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 867 if (err) 868 goto err_table_erp_add; 869 870 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 871 if (err) 872 goto err_region_erp_add; 873 874 erp_table->ops = &erp_multiple_masks_ops; 875 876 return erp; 877 878 err_region_erp_add: 879 mlxsw_sp_acl_erp_table_erp_del(erp); 880 err_table_erp_add: 881 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 882 err_erp_index_get: 883 mlxsw_sp_acl_erp_generic_destroy(erp); 884 return ERR_PTR(err); 885 } 886 887 static void 888 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 889 struct mlxsw_sp_acl_erp *erp) 890 { 891 if (erp->key.ctcam) 892 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 893 894 mlxsw_sp_acl_erp_region_erp_del(erp); 895 mlxsw_sp_acl_erp_table_erp_del(erp); 896 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 897 mlxsw_sp_acl_erp_generic_destroy(erp); 898 899 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 && 900 erp_table->num_deltas == 0) 901 erp_table->ops = &erp_two_masks_ops; 902 } 903 904 static struct mlxsw_sp_acl_erp * 905 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 906 struct mlxsw_sp_acl_erp_key *key) 907 { 908 struct mlxsw_sp_acl_erp *erp; 909 int err; 910 911 if (key->ctcam) 912 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 913 914 /* Transition to use eRP table instead of master mask */ 915 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 916 if (err) 917 return ERR_PTR(err); 918 919 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 920 if (IS_ERR(erp)) { 921 err = PTR_ERR(erp); 922 goto err_erp_create; 923 } 924 925 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 926 if (err) 927 goto err_erp_index_get; 928 929 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 930 if (err) 931 goto err_table_erp_add; 932 933 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 934 if (err) 935 goto err_region_erp_add; 936 937 erp_table->ops = &erp_two_masks_ops; 938 939 return erp; 940 941 err_region_erp_add: 942 mlxsw_sp_acl_erp_table_erp_del(erp); 943 err_table_erp_add: 944 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 945 err_erp_index_get: 946 mlxsw_sp_acl_erp_generic_destroy(erp); 947 err_erp_create: 948 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 949 return ERR_PTR(err); 950 } 951 952 static void 953 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 954 struct mlxsw_sp_acl_erp *erp) 955 { 956 if (erp->key.ctcam) 957 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 958 959 mlxsw_sp_acl_erp_region_erp_del(erp); 960 mlxsw_sp_acl_erp_table_erp_del(erp); 961 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 962 mlxsw_sp_acl_erp_generic_destroy(erp); 963 /* Transition to use master mask instead of eRP table */ 964 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 965 966 erp_table->ops = &erp_single_mask_ops; 967 } 968 969 static struct mlxsw_sp_acl_erp * 970 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 971 struct mlxsw_sp_acl_erp_key *key) 972 { 973 struct mlxsw_sp_acl_erp *erp; 974 975 if (key->ctcam) 976 return ERR_PTR(-EINVAL); 977 978 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 979 if (IS_ERR(erp)) 980 return erp; 981 982 erp_table->ops = &erp_single_mask_ops; 983 984 return erp; 985 } 986 987 static void 988 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 989 struct mlxsw_sp_acl_erp *erp) 990 { 991 mlxsw_sp_acl_erp_generic_destroy(erp); 992 erp_table->ops = &erp_no_mask_ops; 993 } 994 995 static void 996 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 997 struct mlxsw_sp_acl_erp *erp) 998 { 999 WARN_ON(1); 1000 } 1001 1002 struct mlxsw_sp_acl_erp_mask * 1003 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion, 1004 const char *mask, bool ctcam) 1005 { 1006 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1007 struct mlxsw_sp_acl_erp_key key; 1008 struct objagg_obj *objagg_obj; 1009 1010 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN); 1011 key.ctcam = ctcam; 1012 mutex_lock(&erp_table->objagg_lock); 1013 objagg_obj = objagg_obj_get(erp_table->objagg, &key); 1014 mutex_unlock(&erp_table->objagg_lock); 1015 if (IS_ERR(objagg_obj)) 1016 return ERR_CAST(objagg_obj); 1017 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj; 1018 } 1019 1020 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion, 1021 struct mlxsw_sp_acl_erp_mask *erp_mask) 1022 { 1023 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1024 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1025 1026 mutex_lock(&erp_table->objagg_lock); 1027 objagg_obj_put(erp_table->objagg, objagg_obj); 1028 mutex_unlock(&erp_table->objagg_lock); 1029 } 1030 1031 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp, 1032 struct mlxsw_sp_acl_atcam_region *aregion, 1033 struct mlxsw_sp_acl_erp_mask *erp_mask, 1034 struct mlxsw_sp_acl_atcam_entry *aentry) 1035 { 1036 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1037 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1038 unsigned int erp_bank; 1039 1040 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1041 return 0; 1042 1043 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1044 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp, 1045 erp->erp_table->erp_core->bf, 1046 aregion, erp_bank, aentry); 1047 } 1048 1049 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp, 1050 struct mlxsw_sp_acl_atcam_region *aregion, 1051 struct mlxsw_sp_acl_erp_mask *erp_mask, 1052 struct mlxsw_sp_acl_atcam_entry *aentry) 1053 { 1054 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1055 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1056 unsigned int erp_bank; 1057 1058 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1059 return; 1060 1061 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1062 mlxsw_sp_acl_bf_entry_del(mlxsw_sp, 1063 erp->erp_table->erp_core->bf, 1064 aregion, erp_bank, aentry); 1065 } 1066 1067 bool 1068 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1069 { 1070 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1071 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj); 1072 1073 return key->ctcam; 1074 } 1075 1076 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1077 { 1078 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1079 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1080 1081 return erp->id; 1082 } 1083 1084 struct mlxsw_sp_acl_erp_delta { 1085 struct mlxsw_sp_acl_erp_key key; 1086 u16 start; 1087 u8 mask; 1088 }; 1089 1090 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta) 1091 { 1092 return delta->start; 1093 } 1094 1095 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta) 1096 { 1097 return delta->mask; 1098 } 1099 1100 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta, 1101 const char *enc_key) 1102 { 1103 u16 start = delta->start; 1104 u8 mask = delta->mask; 1105 u16 tmp; 1106 1107 if (!mask) 1108 return 0; 1109 1110 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)]; 1111 if (start / 8 + 1 < __MASK_LEN) 1112 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8; 1113 tmp >>= start % 8; 1114 tmp &= mask; 1115 return tmp; 1116 } 1117 1118 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta, 1119 const char *enc_key) 1120 { 1121 u16 start = delta->start; 1122 u8 mask = delta->mask; 1123 unsigned char *byte; 1124 u16 tmp; 1125 1126 tmp = mask; 1127 tmp <<= start % 8; 1128 tmp = ~tmp; 1129 1130 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)]; 1131 *byte &= tmp & 0xff; 1132 if (start / 8 + 1 < __MASK_LEN) { 1133 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)]; 1134 *byte &= (tmp >> 8) & 0xff; 1135 } 1136 } 1137 1138 static const struct mlxsw_sp_acl_erp_delta 1139 mlxsw_sp_acl_erp_delta_default = {}; 1140 1141 const struct mlxsw_sp_acl_erp_delta * 1142 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1143 { 1144 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1145 const struct mlxsw_sp_acl_erp_delta *delta; 1146 1147 delta = objagg_obj_delta_priv(objagg_obj); 1148 if (!delta) 1149 delta = &mlxsw_sp_acl_erp_delta_default; 1150 return delta; 1151 } 1152 1153 static int 1154 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key, 1155 const struct mlxsw_sp_acl_erp_key *key, 1156 u16 *delta_start, u8 *delta_mask) 1157 { 1158 int offset = 0; 1159 int si = -1; 1160 u16 pmask; 1161 u16 mask; 1162 int i; 1163 1164 /* The difference between 2 masks can be up to 8 consecutive bits. */ 1165 for (i = 0; i < __MASK_LEN; i++) { 1166 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)]) 1167 continue; 1168 if (si == -1) 1169 si = i; 1170 else if (si != i - 1) 1171 return -EINVAL; 1172 } 1173 if (si == -1) { 1174 /* The masks are the same, this can happen in case eRPs with 1175 * the same mask were created in both A-TCAM and C-TCAM. 1176 * The only possible condition under which this can happen 1177 * is identical rule insertion. Delta is not possible here. 1178 */ 1179 return -EINVAL; 1180 } 1181 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)]; 1182 mask = (unsigned char) key->mask[__MASK_IDX(si)]; 1183 if (si + 1 < __MASK_LEN) { 1184 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8; 1185 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8; 1186 } 1187 1188 if ((pmask ^ mask) & pmask) 1189 return -EINVAL; 1190 mask &= ~pmask; 1191 while (!(mask & (1 << offset))) 1192 offset++; 1193 while (!(mask & 1)) 1194 mask >>= 1; 1195 if (mask & 0xff00) 1196 return -EINVAL; 1197 1198 *delta_start = si * 8 + offset; 1199 *delta_mask = mask; 1200 1201 return 0; 1202 } 1203 1204 static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj, 1205 const void *obj) 1206 { 1207 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1208 const struct mlxsw_sp_acl_erp_key *key = obj; 1209 u16 delta_start; 1210 u8 delta_mask; 1211 int err; 1212 1213 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1214 &delta_start, &delta_mask); 1215 return err ? false : true; 1216 } 1217 1218 static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2) 1219 { 1220 const struct mlxsw_sp_acl_erp_key *key1 = obj1; 1221 const struct mlxsw_sp_acl_erp_key *key2 = obj2; 1222 1223 /* For hints purposes, two objects are considered equal 1224 * in case the masks are the same. Does not matter what 1225 * the "ctcam" value is. 1226 */ 1227 return memcmp(key1->mask, key2->mask, sizeof(key1->mask)); 1228 } 1229 1230 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj, 1231 void *obj) 1232 { 1233 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1234 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1235 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1236 struct mlxsw_sp_acl_erp_key *key = obj; 1237 struct mlxsw_sp_acl_erp_delta *delta; 1238 u16 delta_start; 1239 u8 delta_mask; 1240 int err; 1241 1242 if (parent_key->ctcam || key->ctcam) 1243 return ERR_PTR(-EINVAL); 1244 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1245 &delta_start, &delta_mask); 1246 if (err) 1247 return ERR_PTR(-EINVAL); 1248 1249 delta = kzalloc(sizeof(*delta), GFP_KERNEL); 1250 if (!delta) 1251 return ERR_PTR(-ENOMEM); 1252 delta->start = delta_start; 1253 delta->mask = delta_mask; 1254 1255 err = mlxsw_sp_acl_erp_delta_inc(erp_table); 1256 if (err) 1257 goto err_erp_delta_inc; 1258 1259 memcpy(&delta->key, key, sizeof(*key)); 1260 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key); 1261 if (err) 1262 goto err_master_mask_set; 1263 1264 return delta; 1265 1266 err_master_mask_set: 1267 mlxsw_sp_acl_erp_delta_dec(erp_table); 1268 err_erp_delta_inc: 1269 kfree(delta); 1270 return ERR_PTR(err); 1271 } 1272 1273 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv) 1274 { 1275 struct mlxsw_sp_acl_erp_delta *delta = delta_priv; 1276 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1277 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1278 1279 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key); 1280 mlxsw_sp_acl_erp_delta_dec(erp_table); 1281 kfree(delta); 1282 } 1283 1284 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj, 1285 unsigned int root_id) 1286 { 1287 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1288 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1289 struct mlxsw_sp_acl_erp_key *key = obj; 1290 1291 if (!key->ctcam && 1292 root_id != OBJAGG_OBJ_ROOT_ID_INVALID && 1293 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION) 1294 return ERR_PTR(-ENOBUFS); 1295 return erp_table->ops->erp_create(erp_table, key); 1296 } 1297 1298 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv) 1299 { 1300 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1301 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1302 1303 erp_table->ops->erp_destroy(erp_table, root_priv); 1304 } 1305 1306 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = { 1307 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key), 1308 .delta_check = mlxsw_sp_acl_erp_delta_check, 1309 .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp, 1310 .delta_create = mlxsw_sp_acl_erp_delta_create, 1311 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy, 1312 .root_create = mlxsw_sp_acl_erp_root_create, 1313 .root_destroy = mlxsw_sp_acl_erp_root_destroy, 1314 }; 1315 1316 static struct mlxsw_sp_acl_erp_table * 1317 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion, 1318 struct objagg_hints *hints) 1319 { 1320 struct mlxsw_sp_acl_erp_table *erp_table; 1321 int err; 1322 1323 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL); 1324 if (!erp_table) 1325 return ERR_PTR(-ENOMEM); 1326 1327 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops, 1328 hints, aregion); 1329 if (IS_ERR(erp_table->objagg)) { 1330 err = PTR_ERR(erp_table->objagg); 1331 goto err_objagg_create; 1332 } 1333 1334 erp_table->erp_core = aregion->atcam->erp_core; 1335 erp_table->ops = &erp_no_mask_ops; 1336 INIT_LIST_HEAD(&erp_table->atcam_erps_list); 1337 erp_table->aregion = aregion; 1338 mutex_init(&erp_table->objagg_lock); 1339 1340 return erp_table; 1341 1342 err_objagg_create: 1343 kfree(erp_table); 1344 return ERR_PTR(err); 1345 } 1346 1347 static void 1348 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table) 1349 { 1350 WARN_ON(!list_empty(&erp_table->atcam_erps_list)); 1351 mutex_destroy(&erp_table->objagg_lock); 1352 objagg_destroy(erp_table->objagg); 1353 kfree(erp_table); 1354 } 1355 1356 static int 1357 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion) 1358 { 1359 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1360 char percr_pl[MLXSW_REG_PERCR_LEN]; 1361 1362 mlxsw_reg_percr_pack(percr_pl, aregion->region->id); 1363 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 1364 } 1365 1366 static int 1367 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion) 1368 { 1369 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1370 char pererp_pl[MLXSW_REG_PERERP_LEN]; 1371 1372 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0, 1373 0, 0); 1374 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 1375 } 1376 1377 static int 1378 mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp, 1379 struct mlxsw_sp_acl_atcam_region *aregion, 1380 struct objagg_hints *hints, bool *p_rehash_needed) 1381 { 1382 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1383 const struct objagg_stats *ostats; 1384 const struct objagg_stats *hstats; 1385 int err; 1386 1387 *p_rehash_needed = false; 1388 1389 mutex_lock(&erp_table->objagg_lock); 1390 ostats = objagg_stats_get(erp_table->objagg); 1391 mutex_unlock(&erp_table->objagg_lock); 1392 if (IS_ERR(ostats)) { 1393 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n"); 1394 return PTR_ERR(ostats); 1395 } 1396 1397 hstats = objagg_hints_stats_get(hints); 1398 if (IS_ERR(hstats)) { 1399 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n"); 1400 err = PTR_ERR(hstats); 1401 goto err_hints_stats_get; 1402 } 1403 1404 /* Very basic criterion for now. */ 1405 if (hstats->root_count < ostats->root_count) 1406 *p_rehash_needed = true; 1407 1408 err = 0; 1409 1410 objagg_stats_put(hstats); 1411 err_hints_stats_get: 1412 objagg_stats_put(ostats); 1413 return err; 1414 } 1415 1416 void * 1417 mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion) 1418 { 1419 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1420 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1421 struct objagg_hints *hints; 1422 bool rehash_needed; 1423 int err; 1424 1425 mutex_lock(&erp_table->objagg_lock); 1426 hints = objagg_hints_get(erp_table->objagg, 1427 OBJAGG_OPT_ALGO_SIMPLE_GREEDY); 1428 mutex_unlock(&erp_table->objagg_lock); 1429 if (IS_ERR(hints)) { 1430 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n"); 1431 return ERR_CAST(hints); 1432 } 1433 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints, 1434 &rehash_needed); 1435 if (err) 1436 goto errout; 1437 1438 if (!rehash_needed) { 1439 err = -EAGAIN; 1440 goto errout; 1441 } 1442 return hints; 1443 1444 errout: 1445 objagg_hints_put(hints); 1446 return ERR_PTR(err); 1447 } 1448 1449 void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv) 1450 { 1451 struct objagg_hints *hints = hints_priv; 1452 1453 objagg_hints_put(hints); 1454 } 1455 1456 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion, 1457 void *hints_priv) 1458 { 1459 struct mlxsw_sp_acl_erp_table *erp_table; 1460 struct objagg_hints *hints = hints_priv; 1461 int err; 1462 1463 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints); 1464 if (IS_ERR(erp_table)) 1465 return PTR_ERR(erp_table); 1466 aregion->erp_table = erp_table; 1467 1468 /* Initialize the region's master mask to all zeroes */ 1469 err = mlxsw_sp_acl_erp_master_mask_init(aregion); 1470 if (err) 1471 goto err_erp_master_mask_init; 1472 1473 /* Initialize the region to not use the eRP table */ 1474 err = mlxsw_sp_acl_erp_region_param_init(aregion); 1475 if (err) 1476 goto err_erp_region_param_init; 1477 1478 return 0; 1479 1480 err_erp_region_param_init: 1481 err_erp_master_mask_init: 1482 mlxsw_sp_acl_erp_table_destroy(erp_table); 1483 return err; 1484 } 1485 1486 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 1487 { 1488 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table); 1489 } 1490 1491 static int 1492 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp, 1493 struct mlxsw_sp_acl_erp_core *erp_core) 1494 { 1495 unsigned int size; 1496 1497 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) || 1498 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) || 1499 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) || 1500 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB)) 1501 return -EIO; 1502 1503 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB); 1504 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size; 1505 1506 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB); 1507 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size; 1508 1509 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB); 1510 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size; 1511 1512 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB); 1513 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size; 1514 1515 return 0; 1516 } 1517 1518 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp, 1519 struct mlxsw_sp_acl_erp_core *erp_core) 1520 { 1521 unsigned int erpt_bank_size; 1522 int err; 1523 1524 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) || 1525 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS)) 1526 return -EIO; 1527 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1528 ACL_MAX_ERPT_BANK_SIZE); 1529 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1530 ACL_MAX_ERPT_BANKS); 1531 1532 erp_core->erp_tables = gen_pool_create(0, -1); 1533 if (!erp_core->erp_tables) 1534 return -ENOMEM; 1535 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL); 1536 1537 err = gen_pool_add(erp_core->erp_tables, 1538 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size, 1539 -1); 1540 if (err) 1541 goto err_gen_pool_add; 1542 1543 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks); 1544 if (IS_ERR(erp_core->bf)) { 1545 err = PTR_ERR(erp_core->bf); 1546 goto err_bf_init; 1547 } 1548 1549 /* Different regions require masks of different sizes */ 1550 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core); 1551 if (err) 1552 goto err_erp_tables_sizes_query; 1553 1554 return 0; 1555 1556 err_erp_tables_sizes_query: 1557 mlxsw_sp_acl_bf_fini(erp_core->bf); 1558 err_bf_init: 1559 err_gen_pool_add: 1560 gen_pool_destroy(erp_core->erp_tables); 1561 return err; 1562 } 1563 1564 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp, 1565 struct mlxsw_sp_acl_erp_core *erp_core) 1566 { 1567 mlxsw_sp_acl_bf_fini(erp_core->bf); 1568 gen_pool_destroy(erp_core->erp_tables); 1569 } 1570 1571 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp, 1572 struct mlxsw_sp_acl_atcam *atcam) 1573 { 1574 struct mlxsw_sp_acl_erp_core *erp_core; 1575 int err; 1576 1577 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL); 1578 if (!erp_core) 1579 return -ENOMEM; 1580 erp_core->mlxsw_sp = mlxsw_sp; 1581 atcam->erp_core = erp_core; 1582 1583 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core); 1584 if (err) 1585 goto err_erp_tables_init; 1586 1587 return 0; 1588 1589 err_erp_tables_init: 1590 kfree(erp_core); 1591 return err; 1592 } 1593 1594 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp, 1595 struct mlxsw_sp_acl_atcam *atcam) 1596 { 1597 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core); 1598 kfree(atcam->erp_core); 1599 } 1600