1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/slab.h> 6 #include <linux/list.h> 7 #include <linux/errno.h> 8 9 #include "item.h" 10 #include "core_acl_flex_keys.h" 11 12 struct mlxsw_afk { 13 struct list_head key_info_list; 14 unsigned int max_blocks; 15 const struct mlxsw_afk_ops *ops; 16 const struct mlxsw_afk_block *blocks; 17 unsigned int blocks_count; 18 }; 19 20 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk) 21 { 22 int i; 23 int j; 24 25 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 26 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 27 28 for (j = 0; j < block->instances_count; j++) { 29 struct mlxsw_afk_element_inst *elinst; 30 31 elinst = &block->instances[j]; 32 if (elinst->type != elinst->info->type || 33 (!elinst->avoid_size_check && 34 elinst->item.size.bits != 35 elinst->info->item.size.bits)) 36 return false; 37 } 38 } 39 return true; 40 } 41 42 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks, 43 const struct mlxsw_afk_ops *ops) 44 { 45 struct mlxsw_afk *mlxsw_afk; 46 47 mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL); 48 if (!mlxsw_afk) 49 return NULL; 50 INIT_LIST_HEAD(&mlxsw_afk->key_info_list); 51 mlxsw_afk->max_blocks = max_blocks; 52 mlxsw_afk->ops = ops; 53 mlxsw_afk->blocks = ops->blocks; 54 mlxsw_afk->blocks_count = ops->blocks_count; 55 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk)); 56 return mlxsw_afk; 57 } 58 EXPORT_SYMBOL(mlxsw_afk_create); 59 60 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk) 61 { 62 WARN_ON(!list_empty(&mlxsw_afk->key_info_list)); 63 kfree(mlxsw_afk); 64 } 65 EXPORT_SYMBOL(mlxsw_afk_destroy); 66 67 struct mlxsw_afk_key_info { 68 struct list_head list; 69 unsigned int ref_count; 70 unsigned int blocks_count; 71 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value 72 * is index inside "blocks" 73 */ 74 struct mlxsw_afk_element_usage elusage; 75 const struct mlxsw_afk_block *blocks[0]; 76 }; 77 78 static bool 79 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info, 80 struct mlxsw_afk_element_usage *elusage) 81 { 82 return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0; 83 } 84 85 static struct mlxsw_afk_key_info * 86 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, 87 struct mlxsw_afk_element_usage *elusage) 88 { 89 struct mlxsw_afk_key_info *key_info; 90 91 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) { 92 if (mlxsw_afk_key_info_elements_eq(key_info, elusage)) 93 return key_info; 94 } 95 return NULL; 96 } 97 98 struct mlxsw_afk_picker { 99 struct { 100 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); 101 unsigned int total; 102 } hits[0]; 103 }; 104 105 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk, 106 struct mlxsw_afk_picker *picker, 107 enum mlxsw_afk_element element) 108 { 109 int i; 110 int j; 111 112 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 113 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 114 115 for (j = 0; j < block->instances_count; j++) { 116 struct mlxsw_afk_element_inst *elinst; 117 118 elinst = &block->instances[j]; 119 if (elinst->info->element == element) { 120 __set_bit(element, picker->hits[i].element); 121 picker->hits[i].total++; 122 } 123 } 124 } 125 } 126 127 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk, 128 struct mlxsw_afk_picker *picker, 129 int block_index) 130 { 131 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX); 132 int i; 133 int j; 134 135 memcpy(&hits_element, &picker->hits[block_index].element, 136 sizeof(hits_element)); 137 138 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 139 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) { 140 if (__test_and_clear_bit(j, picker->hits[i].element)) 141 picker->hits[i].total--; 142 } 143 } 144 } 145 146 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk, 147 struct mlxsw_afk_picker *picker) 148 { 149 int most_index = -EINVAL; /* Should never happen to return this */ 150 int most_hits = 0; 151 int i; 152 153 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 154 if (picker->hits[i].total > most_hits) { 155 most_hits = picker->hits[i].total; 156 most_index = i; 157 } 158 } 159 return most_index; 160 } 161 162 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, 163 struct mlxsw_afk_picker *picker, 164 int block_index, 165 struct mlxsw_afk_key_info *key_info) 166 { 167 enum mlxsw_afk_element element; 168 169 if (key_info->blocks_count == mlxsw_afk->max_blocks) 170 return -EINVAL; 171 172 for_each_set_bit(element, picker->hits[block_index].element, 173 MLXSW_AFK_ELEMENT_MAX) { 174 key_info->element_to_block[element] = key_info->blocks_count; 175 mlxsw_afk_element_usage_add(&key_info->elusage, element); 176 } 177 178 key_info->blocks[key_info->blocks_count] = 179 &mlxsw_afk->blocks[block_index]; 180 key_info->blocks_count++; 181 return 0; 182 } 183 184 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, 185 struct mlxsw_afk_key_info *key_info, 186 struct mlxsw_afk_element_usage *elusage) 187 { 188 struct mlxsw_afk_picker *picker; 189 enum mlxsw_afk_element element; 190 size_t alloc_size; 191 int err; 192 193 alloc_size = sizeof(picker->hits[0]) * mlxsw_afk->blocks_count; 194 picker = kzalloc(alloc_size, GFP_KERNEL); 195 if (!picker) 196 return -ENOMEM; 197 198 /* Since the same elements could be present in multiple blocks, 199 * we must find out optimal block list in order to make the 200 * block count as low as possible. 201 * 202 * First, we count hits. We go over all available blocks and count 203 * how many of requested elements are covered by each. 204 * 205 * Then in loop, we find block with most hits and add it to 206 * output key_info. Then we have to subtract this block hits so 207 * the next iteration will find most suitable block for 208 * the rest of requested elements. 209 */ 210 211 mlxsw_afk_element_usage_for_each(element, elusage) 212 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element); 213 214 do { 215 int block_index; 216 217 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker); 218 if (block_index < 0) { 219 err = block_index; 220 goto out; 221 } 222 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, 223 block_index, key_info); 224 if (err) 225 goto out; 226 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); 227 } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); 228 229 err = 0; 230 out: 231 kfree(picker); 232 return err; 233 } 234 235 static struct mlxsw_afk_key_info * 236 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk, 237 struct mlxsw_afk_element_usage *elusage) 238 { 239 struct mlxsw_afk_key_info *key_info; 240 int err; 241 242 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks), 243 GFP_KERNEL); 244 if (!key_info) 245 return ERR_PTR(-ENOMEM); 246 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage); 247 if (err) 248 goto err_picker; 249 list_add(&key_info->list, &mlxsw_afk->key_info_list); 250 key_info->ref_count = 1; 251 return key_info; 252 253 err_picker: 254 kfree(key_info); 255 return ERR_PTR(err); 256 } 257 258 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info) 259 { 260 list_del(&key_info->list); 261 kfree(key_info); 262 } 263 264 struct mlxsw_afk_key_info * 265 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk, 266 struct mlxsw_afk_element_usage *elusage) 267 { 268 struct mlxsw_afk_key_info *key_info; 269 270 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage); 271 if (key_info) { 272 key_info->ref_count++; 273 return key_info; 274 } 275 return mlxsw_afk_key_info_create(mlxsw_afk, elusage); 276 } 277 EXPORT_SYMBOL(mlxsw_afk_key_info_get); 278 279 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info) 280 { 281 if (--key_info->ref_count) 282 return; 283 mlxsw_afk_key_info_destroy(key_info); 284 } 285 EXPORT_SYMBOL(mlxsw_afk_key_info_put); 286 287 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info, 288 struct mlxsw_afk_element_usage *elusage) 289 { 290 return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage); 291 } 292 EXPORT_SYMBOL(mlxsw_afk_key_info_subset); 293 294 static const struct mlxsw_afk_element_inst * 295 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block, 296 enum mlxsw_afk_element element) 297 { 298 int i; 299 300 for (i = 0; i < block->instances_count; i++) { 301 struct mlxsw_afk_element_inst *elinst; 302 303 elinst = &block->instances[i]; 304 if (elinst->info->element == element) 305 return elinst; 306 } 307 return NULL; 308 } 309 310 static const struct mlxsw_afk_element_inst * 311 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info, 312 enum mlxsw_afk_element element, 313 int *p_block_index) 314 { 315 const struct mlxsw_afk_element_inst *elinst; 316 const struct mlxsw_afk_block *block; 317 int block_index; 318 319 if (WARN_ON(!test_bit(element, key_info->elusage.usage))) 320 return NULL; 321 block_index = key_info->element_to_block[element]; 322 block = key_info->blocks[block_index]; 323 324 elinst = mlxsw_afk_block_elinst_get(block, element); 325 if (WARN_ON(!elinst)) 326 return NULL; 327 328 *p_block_index = block_index; 329 return elinst; 330 } 331 332 u16 333 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info, 334 int block_index) 335 { 336 return key_info->blocks[block_index]->encoding; 337 } 338 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get); 339 340 unsigned int 341 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info) 342 { 343 return key_info->blocks_count; 344 } 345 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get); 346 347 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values, 348 enum mlxsw_afk_element element, 349 u32 key_value, u32 mask_value) 350 { 351 const struct mlxsw_afk_element_info *elinfo = 352 &mlxsw_afk_element_infos[element]; 353 const struct mlxsw_item *storage_item = &elinfo->item; 354 355 if (!mask_value) 356 return; 357 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32)) 358 return; 359 __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value); 360 __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value); 361 mlxsw_afk_element_usage_add(&values->elusage, element); 362 } 363 EXPORT_SYMBOL(mlxsw_afk_values_add_u32); 364 365 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values, 366 enum mlxsw_afk_element element, 367 const char *key_value, const char *mask_value, 368 unsigned int len) 369 { 370 const struct mlxsw_afk_element_info *elinfo = 371 &mlxsw_afk_element_infos[element]; 372 const struct mlxsw_item *storage_item = &elinfo->item; 373 374 if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */ 375 return; 376 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) || 377 WARN_ON(elinfo->item.size.bytes != len)) 378 return; 379 __mlxsw_item_memcpy_to(values->storage.key, key_value, 380 storage_item, 0); 381 __mlxsw_item_memcpy_to(values->storage.mask, mask_value, 382 storage_item, 0); 383 mlxsw_afk_element_usage_add(&values->elusage, element); 384 } 385 EXPORT_SYMBOL(mlxsw_afk_values_add_buf); 386 387 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item, 388 const struct mlxsw_item *output_item, 389 char *storage, char *output, int diff) 390 { 391 u32 value; 392 393 value = __mlxsw_item_get32(storage, storage_item, 0); 394 __mlxsw_item_set32(output, output_item, 0, value + diff); 395 } 396 397 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item, 398 const struct mlxsw_item *output_item, 399 char *storage, char *output) 400 { 401 char *storage_data = __mlxsw_item_data(storage, storage_item, 0); 402 char *output_data = __mlxsw_item_data(output, output_item, 0); 403 size_t len = output_item->size.bytes; 404 405 memcpy(output_data, storage_data, len); 406 } 407 408 static void 409 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst, 410 char *output, char *storage, int u32_diff) 411 { 412 const struct mlxsw_item *storage_item = &elinst->info->item; 413 const struct mlxsw_item *output_item = &elinst->item; 414 415 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32) 416 mlxsw_sp_afk_encode_u32(storage_item, output_item, 417 storage, output, u32_diff); 418 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF) 419 mlxsw_sp_afk_encode_buf(storage_item, output_item, 420 storage, output); 421 } 422 423 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16 424 425 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk, 426 struct mlxsw_afk_key_info *key_info, 427 struct mlxsw_afk_element_values *values, 428 char *key, char *mask) 429 { 430 unsigned int blocks_count = 431 mlxsw_afk_key_info_blocks_count_get(key_info); 432 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 433 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 434 const struct mlxsw_afk_element_inst *elinst; 435 enum mlxsw_afk_element element; 436 int block_index, i; 437 438 for (i = 0; i < blocks_count; i++) { 439 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 440 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 441 442 mlxsw_afk_element_usage_for_each(element, &values->elusage) { 443 elinst = mlxsw_afk_key_info_elinst_get(key_info, 444 element, 445 &block_index); 446 if (!elinst || block_index != i) 447 continue; 448 449 mlxsw_sp_afk_encode_one(elinst, block_key, 450 values->storage.key, 451 elinst->u32_key_diff); 452 mlxsw_sp_afk_encode_one(elinst, block_mask, 453 values->storage.mask, 0); 454 } 455 456 mlxsw_afk->ops->encode_block(key, i, block_key); 457 mlxsw_afk->ops->encode_block(mask, i, block_mask); 458 } 459 } 460 EXPORT_SYMBOL(mlxsw_afk_encode); 461 462 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key, 463 int block_start, int block_end) 464 { 465 int i; 466 467 for (i = block_start; i <= block_end; i++) 468 mlxsw_afk->ops->clear_block(key, i); 469 } 470 EXPORT_SYMBOL(mlxsw_afk_clear); 471