1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 19 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 20 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 21 22 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 23 #define NPC_HW_TSTAMP_OFFSET 8ULL 24 #define NPC_KEX_CHAN_MASK 0xFFFULL 25 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 26 27 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 28 29 static const char def_pfl_name[] = "default"; 30 31 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 32 int blkaddr, u16 pcifunc); 33 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 34 u16 pcifunc); 35 36 bool is_npc_intf_tx(u8 intf) 37 { 38 return !!(intf & 0x1); 39 } 40 41 bool is_npc_intf_rx(u8 intf) 42 { 43 return !(intf & 0x1); 44 } 45 46 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 47 { 48 struct rvu_hwinfo *hw = rvu->hw; 49 50 return intf < hw->npc_intfs; 51 } 52 53 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 54 { 55 /* Due to a HW issue in these silicon versions, parse nibble enable 56 * configuration has to be identical for both Rx and Tx interfaces. 57 */ 58 if (is_rvu_96xx_B0(rvu)) 59 return nibble_ena; 60 return 0; 61 } 62 63 static int npc_mcam_verify_pf_func(struct rvu *rvu, 64 struct mcam_entry *entry_data, u8 intf, 65 u16 pcifunc) 66 { 67 u16 pf_func, pf_func_mask; 68 69 if (is_npc_intf_rx(intf)) 70 return 0; 71 72 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 73 NPC_KEX_PF_FUNC_MASK; 74 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 75 76 pf_func = be16_to_cpu((__force __be16)pf_func); 77 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 78 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 79 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 80 return -EINVAL; 81 82 return 0; 83 } 84 85 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 86 { 87 int blkaddr; 88 u64 val = 0; 89 90 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 91 if (blkaddr < 0) 92 return; 93 94 /* Config CPI base for the PKIND */ 95 val = pkind | 1ULL << 62; 96 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 97 } 98 99 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 100 { 101 struct npc_pkind *pkind = &rvu->hw->pkind; 102 u32 map; 103 int i; 104 105 for (i = 0; i < pkind->rsrc.max; i++) { 106 map = pkind->pfchan_map[i]; 107 if (((map >> 16) & 0x3F) == pf) 108 return i; 109 } 110 return -1; 111 } 112 113 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 114 115 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 116 { 117 int pkind, blkaddr; 118 u64 val; 119 120 pkind = rvu_npc_get_pkind(rvu, pf); 121 if (pkind < 0) { 122 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 123 return -EINVAL; 124 } 125 126 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 127 if (blkaddr < 0) { 128 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 129 return -EINVAL; 130 } 131 132 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 133 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 134 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 135 if (enable) 136 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 137 NPC_HW_TSTAMP_OFFSET); 138 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 139 140 return 0; 141 } 142 143 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 144 int nixlf) 145 { 146 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 147 struct rvu *rvu = hw->rvu; 148 int blkaddr = 0, max = 0; 149 struct rvu_block *block; 150 struct rvu_pfvf *pfvf; 151 152 pfvf = rvu_get_pfvf(rvu, pcifunc); 153 /* Given a PF/VF and NIX LF number calculate the unicast mcam 154 * entry index based on the NIX block assigned to the PF/VF. 155 */ 156 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 157 while (blkaddr) { 158 if (pfvf->nix_blkaddr == blkaddr) 159 break; 160 block = &rvu->hw->block[blkaddr]; 161 max += block->lf.max; 162 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 163 } 164 165 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 166 } 167 168 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 169 u16 pcifunc, int nixlf, int type) 170 { 171 int pf = rvu_get_pf(pcifunc); 172 int index; 173 174 /* Check if this is for a PF */ 175 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 176 /* Reserved entries exclude PF0 */ 177 pf--; 178 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 179 /* Broadcast address matching entry should be first so 180 * that the packet can be replicated to all VFs. 181 */ 182 if (type == NIXLF_BCAST_ENTRY) 183 return index; 184 else if (type == NIXLF_ALLMULTI_ENTRY) 185 return index + 1; 186 else if (type == NIXLF_PROMISC_ENTRY) 187 return index + 2; 188 } 189 190 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 191 } 192 193 int npc_get_bank(struct npc_mcam *mcam, int index) 194 { 195 int bank = index / mcam->banksize; 196 197 /* 0,1 & 2,3 banks are combined for this keysize */ 198 if (mcam->keysize == NPC_MCAM_KEY_X2) 199 return bank ? 2 : 0; 200 201 return bank; 202 } 203 204 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 205 int blkaddr, int index) 206 { 207 int bank = npc_get_bank(mcam, index); 208 u64 cfg; 209 210 index &= (mcam->banksize - 1); 211 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 212 return (cfg & 1); 213 } 214 215 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 216 int blkaddr, int index, bool enable) 217 { 218 int bank = npc_get_bank(mcam, index); 219 int actbank = bank; 220 221 index &= (mcam->banksize - 1); 222 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 223 rvu_write64(rvu, blkaddr, 224 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 225 enable ? 1 : 0); 226 } 227 } 228 229 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 230 int blkaddr, int index) 231 { 232 int bank = npc_get_bank(mcam, index); 233 int actbank = bank; 234 235 index &= (mcam->banksize - 1); 236 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 237 rvu_write64(rvu, blkaddr, 238 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 239 rvu_write64(rvu, blkaddr, 240 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 241 242 rvu_write64(rvu, blkaddr, 243 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 244 rvu_write64(rvu, blkaddr, 245 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 246 247 rvu_write64(rvu, blkaddr, 248 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 249 rvu_write64(rvu, blkaddr, 250 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 251 } 252 } 253 254 static void npc_get_keyword(struct mcam_entry *entry, int idx, 255 u64 *cam0, u64 *cam1) 256 { 257 u64 kw_mask = 0x00; 258 259 #define CAM_MASK(n) (BIT_ULL(n) - 1) 260 261 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 262 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 263 * 264 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 265 */ 266 switch (idx) { 267 case 0: 268 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 269 *cam1 = entry->kw[0]; 270 kw_mask = entry->kw_mask[0]; 271 break; 272 case 1: 273 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 274 *cam1 = entry->kw[1] & CAM_MASK(48); 275 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 276 break; 277 case 2: 278 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 279 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 280 */ 281 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 282 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 283 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 284 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 285 break; 286 case 3: 287 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 288 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 289 */ 290 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 291 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 292 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 293 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 294 break; 295 case 4: 296 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 297 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 298 */ 299 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 300 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 301 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 302 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 303 break; 304 case 5: 305 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 306 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 307 */ 308 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 309 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 310 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 311 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 312 break; 313 case 6: 314 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 315 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 316 */ 317 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 318 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 319 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 320 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 321 break; 322 case 7: 323 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 324 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 325 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 326 break; 327 } 328 329 *cam1 &= kw_mask; 330 *cam0 = ~*cam1 & kw_mask; 331 } 332 333 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 334 u64 cam0, u64 cam1) 335 { 336 /* Similar to npc_get_keyword, but fills mcam_entry structure from 337 * CAM registers. 338 */ 339 switch (idx) { 340 case 0: 341 entry->kw[0] = cam1; 342 entry->kw_mask[0] = cam1 ^ cam0; 343 break; 344 case 1: 345 entry->kw[1] = cam1; 346 entry->kw_mask[1] = cam1 ^ cam0; 347 break; 348 case 2: 349 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 350 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 351 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 352 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 353 break; 354 case 3: 355 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 356 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 357 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 358 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 359 break; 360 case 4: 361 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 362 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 363 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 364 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 365 break; 366 case 5: 367 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 368 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 369 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 370 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 371 break; 372 case 6: 373 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 374 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 375 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 376 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 377 break; 378 case 7: 379 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 380 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 381 break; 382 } 383 } 384 385 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 386 int blkaddr, u16 pf_func) 387 { 388 int bank, nixlf, index; 389 390 /* get ucast entry rule entry index */ 391 nix_get_nixlf(rvu, pf_func, &nixlf, NULL); 392 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 393 NIXLF_UCAST_ENTRY); 394 bank = npc_get_bank(mcam, index); 395 index &= (mcam->banksize - 1); 396 397 return rvu_read64(rvu, blkaddr, 398 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 399 } 400 401 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 402 int blkaddr, int index, struct mcam_entry *entry, 403 bool *enable) 404 { 405 struct rvu_npc_mcam_rule *rule; 406 u16 owner, target_func; 407 struct rvu_pfvf *pfvf; 408 u64 rx_action; 409 410 owner = mcam->entry2pfvf_map[index]; 411 target_func = (entry->action >> 4) & 0xffff; 412 /* do nothing when target is LBK/PF or owner is not PF */ 413 if (is_pffunc_af(owner) || is_afvf(target_func) || 414 (owner & RVU_PFVF_FUNC_MASK) || 415 !(target_func & RVU_PFVF_FUNC_MASK)) 416 return; 417 418 /* save entry2target_pffunc */ 419 pfvf = rvu_get_pfvf(rvu, target_func); 420 mcam->entry2target_pffunc[index] = target_func; 421 422 /* don't enable rule when nixlf not attached or initialized */ 423 if (!(is_nixlf_attached(rvu, target_func) && 424 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 425 *enable = false; 426 427 /* fix up not needed for the rules added by user(ntuple filters) */ 428 list_for_each_entry(rule, &mcam->mcam_rules, list) { 429 if (rule->entry == index) 430 return; 431 } 432 433 /* copy VF default entry action to the VF mcam entry */ 434 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 435 target_func); 436 if (rx_action) 437 entry->action = rx_action; 438 } 439 440 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 441 int blkaddr, int index, u8 intf, 442 struct mcam_entry *entry, bool enable) 443 { 444 int bank = npc_get_bank(mcam, index); 445 int kw = 0, actbank, actindex; 446 u8 tx_intf_mask = ~intf & 0x3; 447 u8 tx_intf = intf; 448 u64 cam0, cam1; 449 450 actbank = bank; /* Save bank id, to set action later on */ 451 actindex = index; 452 index &= (mcam->banksize - 1); 453 454 /* Disable before mcam entry update */ 455 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 456 457 /* Clear mcam entry to avoid writes being suppressed by NPC */ 458 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 459 460 /* CAM1 takes the comparison value and 461 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 462 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 463 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 464 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 465 */ 466 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 467 /* Interface should be set in all banks */ 468 if (is_npc_intf_tx(intf)) { 469 /* Last bit must be set and rest don't care 470 * for TX interfaces 471 */ 472 tx_intf_mask = 0x1; 473 tx_intf = intf & tx_intf_mask; 474 tx_intf_mask = ~tx_intf & tx_intf_mask; 475 } 476 477 rvu_write64(rvu, blkaddr, 478 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 479 tx_intf); 480 rvu_write64(rvu, blkaddr, 481 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 482 tx_intf_mask); 483 484 /* Set the match key */ 485 npc_get_keyword(entry, kw, &cam0, &cam1); 486 rvu_write64(rvu, blkaddr, 487 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 488 rvu_write64(rvu, blkaddr, 489 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 490 491 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 492 rvu_write64(rvu, blkaddr, 493 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 494 rvu_write64(rvu, blkaddr, 495 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 496 } 497 498 /* PF installing VF rule */ 499 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 500 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 501 502 /* Set 'action' */ 503 rvu_write64(rvu, blkaddr, 504 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 505 506 /* Set TAG 'action' */ 507 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 508 entry->vtag_action); 509 510 /* Enable the entry */ 511 if (enable) 512 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 513 } 514 515 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 516 int blkaddr, u16 src, 517 struct mcam_entry *entry, u8 *intf, u8 *ena) 518 { 519 int sbank = npc_get_bank(mcam, src); 520 int bank, kw = 0; 521 u64 cam0, cam1; 522 523 src &= (mcam->banksize - 1); 524 bank = sbank; 525 526 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 527 cam1 = rvu_read64(rvu, blkaddr, 528 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 529 cam0 = rvu_read64(rvu, blkaddr, 530 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 531 npc_fill_entryword(entry, kw, cam0, cam1); 532 533 cam1 = rvu_read64(rvu, blkaddr, 534 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 535 cam0 = rvu_read64(rvu, blkaddr, 536 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 537 npc_fill_entryword(entry, kw + 1, cam0, cam1); 538 } 539 540 entry->action = rvu_read64(rvu, blkaddr, 541 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 542 entry->vtag_action = 543 rvu_read64(rvu, blkaddr, 544 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 545 *intf = rvu_read64(rvu, blkaddr, 546 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 547 *ena = rvu_read64(rvu, blkaddr, 548 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 549 } 550 551 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 552 int blkaddr, u16 src, u16 dest) 553 { 554 int dbank = npc_get_bank(mcam, dest); 555 int sbank = npc_get_bank(mcam, src); 556 u64 cfg, sreg, dreg; 557 int bank, i; 558 559 src &= (mcam->banksize - 1); 560 dest &= (mcam->banksize - 1); 561 562 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 563 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 564 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 565 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 566 for (i = 0; i < 6; i++) { 567 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 568 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 569 } 570 } 571 572 /* Copy action */ 573 cfg = rvu_read64(rvu, blkaddr, 574 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 575 rvu_write64(rvu, blkaddr, 576 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 577 578 /* Copy TAG action */ 579 cfg = rvu_read64(rvu, blkaddr, 580 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 581 rvu_write64(rvu, blkaddr, 582 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 583 584 /* Enable or disable */ 585 cfg = rvu_read64(rvu, blkaddr, 586 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 587 rvu_write64(rvu, blkaddr, 588 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 589 } 590 591 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 592 int blkaddr, int index) 593 { 594 int bank = npc_get_bank(mcam, index); 595 596 index &= (mcam->banksize - 1); 597 return rvu_read64(rvu, blkaddr, 598 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 599 } 600 601 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 602 int nixlf, u64 chan, u8 *mac_addr) 603 { 604 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 605 struct npc_install_flow_req req = { 0 }; 606 struct npc_install_flow_rsp rsp = { 0 }; 607 struct npc_mcam *mcam = &rvu->hw->mcam; 608 struct nix_rx_action action = { 0 }; 609 int blkaddr, index; 610 611 /* AF's and SDP VFs work in promiscuous mode */ 612 if (is_afvf(pcifunc) || is_sdp_vf(pcifunc)) 613 return; 614 615 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 616 if (blkaddr < 0) 617 return; 618 619 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 620 nixlf, NIXLF_UCAST_ENTRY); 621 622 /* Don't change the action if entry is already enabled 623 * Otherwise RSS action may get overwritten. 624 */ 625 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 626 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 627 blkaddr, index); 628 } else { 629 action.op = NIX_RX_ACTIONOP_UCAST; 630 action.pf_func = pcifunc; 631 } 632 633 req.default_rule = 1; 634 ether_addr_copy(req.packet.dmac, mac_addr); 635 eth_broadcast_addr((u8 *)&req.mask.dmac); 636 req.features = BIT_ULL(NPC_DMAC); 637 req.channel = chan; 638 req.chan_mask = 0xFFFU; 639 req.intf = pfvf->nix_rx_intf; 640 req.op = action.op; 641 req.hdr.pcifunc = 0; /* AF is requester */ 642 req.vf = action.pf_func; 643 req.index = action.index; 644 req.match_id = action.match_id; 645 req.flow_key_alg = action.flow_key_alg; 646 647 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 648 } 649 650 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 651 int nixlf, u64 chan, u8 chan_cnt) 652 { 653 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 654 struct npc_install_flow_req req = { 0 }; 655 struct npc_install_flow_rsp rsp = { 0 }; 656 struct npc_mcam *mcam = &rvu->hw->mcam; 657 struct rvu_hwinfo *hw = rvu->hw; 658 int blkaddr, ucast_idx, index; 659 struct nix_rx_action action = { 0 }; 660 u64 relaxed_mask; 661 662 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 663 return; 664 665 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 666 if (blkaddr < 0) 667 return; 668 669 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 670 nixlf, NIXLF_PROMISC_ENTRY); 671 672 if (is_cgx_vf(rvu, pcifunc)) 673 index = npc_get_nixlf_mcam_index(mcam, 674 pcifunc & ~RVU_PFVF_FUNC_MASK, 675 nixlf, NIXLF_PROMISC_ENTRY); 676 677 /* If the corresponding PF's ucast action is RSS, 678 * use the same action for promisc also 679 */ 680 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 681 nixlf, NIXLF_UCAST_ENTRY); 682 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 683 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 684 blkaddr, ucast_idx); 685 686 if (action.op != NIX_RX_ACTIONOP_RSS) { 687 *(u64 *)&action = 0; 688 action.op = NIX_RX_ACTIONOP_UCAST; 689 } 690 691 /* RX_ACTION set to MCAST for CGX PF's */ 692 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 693 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { 694 *(u64 *)&action = 0; 695 action.op = NIX_RX_ACTIONOP_MCAST; 696 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 697 action.index = pfvf->promisc_mce_idx; 698 } 699 700 /* For cn10k the upper two bits of the channel number are 701 * cpt channel number. with masking out these bits in the 702 * mcam entry, same entry used for NIX will allow packets 703 * received from cpt for parsing. 704 */ 705 if (!is_rvu_otx2(rvu)) { 706 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 707 } else { 708 req.chan_mask = 0xFFFU; 709 } 710 711 if (chan_cnt > 1) { 712 if (!is_power_of_2(chan_cnt)) { 713 dev_err(rvu->dev, 714 "%s: channel count more than 1, must be power of 2\n", __func__); 715 return; 716 } 717 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 718 ilog2(chan_cnt)); 719 req.chan_mask &= relaxed_mask; 720 } 721 722 req.channel = chan; 723 req.intf = pfvf->nix_rx_intf; 724 req.entry = index; 725 req.op = action.op; 726 req.hdr.pcifunc = 0; /* AF is requester */ 727 req.vf = pcifunc; 728 req.index = action.index; 729 req.match_id = action.match_id; 730 req.flow_key_alg = action.flow_key_alg; 731 732 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 733 } 734 735 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 736 int nixlf, bool enable) 737 { 738 struct npc_mcam *mcam = &rvu->hw->mcam; 739 int blkaddr, index; 740 741 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 742 if (blkaddr < 0) 743 return; 744 745 /* Get 'pcifunc' of PF device */ 746 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 747 748 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 749 nixlf, NIXLF_PROMISC_ENTRY); 750 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 751 } 752 753 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 754 int nixlf, u64 chan) 755 { 756 struct rvu_pfvf *pfvf; 757 struct npc_install_flow_req req = { 0 }; 758 struct npc_install_flow_rsp rsp = { 0 }; 759 struct npc_mcam *mcam = &rvu->hw->mcam; 760 struct rvu_hwinfo *hw = rvu->hw; 761 int blkaddr, index; 762 763 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 764 if (blkaddr < 0) 765 return; 766 767 /* Skip LBK VFs */ 768 if (is_afvf(pcifunc)) 769 return; 770 771 /* If pkt replication is not supported, 772 * then only PF is allowed to add a bcast match entry. 773 */ 774 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 775 return; 776 777 /* Get 'pcifunc' of PF device */ 778 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 779 pfvf = rvu_get_pfvf(rvu, pcifunc); 780 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 781 nixlf, NIXLF_BCAST_ENTRY); 782 783 if (!hw->cap.nix_rx_multicast) { 784 /* Early silicon doesn't support pkt replication, 785 * so install entry with UCAST action, so that PF 786 * receives all broadcast packets. 787 */ 788 req.op = NIX_RX_ACTIONOP_UCAST; 789 } else { 790 req.op = NIX_RX_ACTIONOP_MCAST; 791 req.index = pfvf->bcast_mce_idx; 792 } 793 794 eth_broadcast_addr((u8 *)&req.packet.dmac); 795 eth_broadcast_addr((u8 *)&req.mask.dmac); 796 req.features = BIT_ULL(NPC_DMAC); 797 req.channel = chan; 798 req.chan_mask = 0xFFFU; 799 req.intf = pfvf->nix_rx_intf; 800 req.entry = index; 801 req.hdr.pcifunc = 0; /* AF is requester */ 802 req.vf = pcifunc; 803 804 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 805 } 806 807 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 808 bool enable) 809 { 810 struct npc_mcam *mcam = &rvu->hw->mcam; 811 int blkaddr, index; 812 813 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 814 if (blkaddr < 0) 815 return; 816 817 /* Get 'pcifunc' of PF device */ 818 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 819 820 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 821 NIXLF_BCAST_ENTRY); 822 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 823 } 824 825 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 826 u64 chan) 827 { 828 struct npc_install_flow_req req = { 0 }; 829 struct npc_install_flow_rsp rsp = { 0 }; 830 struct npc_mcam *mcam = &rvu->hw->mcam; 831 struct rvu_hwinfo *hw = rvu->hw; 832 int blkaddr, ucast_idx, index; 833 u8 mac_addr[ETH_ALEN] = { 0 }; 834 struct nix_rx_action action = { 0 }; 835 struct rvu_pfvf *pfvf; 836 u16 vf_func; 837 838 /* Only CGX PF/VF can add allmulticast entry */ 839 if (is_afvf(pcifunc) && is_sdp_vf(pcifunc)) 840 return; 841 842 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 843 if (blkaddr < 0) 844 return; 845 846 /* Get 'pcifunc' of PF device */ 847 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 848 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 849 pfvf = rvu_get_pfvf(rvu, pcifunc); 850 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 851 nixlf, NIXLF_ALLMULTI_ENTRY); 852 853 /* If the corresponding PF's ucast action is RSS, 854 * use the same action for multicast entry also 855 */ 856 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 857 nixlf, NIXLF_UCAST_ENTRY); 858 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 859 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 860 blkaddr, ucast_idx); 861 862 if (action.op != NIX_RX_ACTIONOP_RSS) { 863 *(u64 *)&action = 0; 864 action.op = NIX_RX_ACTIONOP_UCAST; 865 action.pf_func = pcifunc; 866 } 867 868 /* RX_ACTION set to MCAST for CGX PF's */ 869 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 870 *(u64 *)&action = 0; 871 action.op = NIX_RX_ACTIONOP_MCAST; 872 action.index = pfvf->mcast_mce_idx; 873 } 874 875 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 876 ether_addr_copy(req.packet.dmac, mac_addr); 877 ether_addr_copy(req.mask.dmac, mac_addr); 878 req.features = BIT_ULL(NPC_DMAC); 879 880 /* For cn10k the upper two bits of the channel number are 881 * cpt channel number. with masking out these bits in the 882 * mcam entry, same entry used for NIX will allow packets 883 * received from cpt for parsing. 884 */ 885 if (!is_rvu_otx2(rvu)) 886 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 887 else 888 req.chan_mask = 0xFFFU; 889 890 req.channel = chan; 891 req.intf = pfvf->nix_rx_intf; 892 req.entry = index; 893 req.op = action.op; 894 req.hdr.pcifunc = 0; /* AF is requester */ 895 req.vf = pcifunc | vf_func; 896 req.index = action.index; 897 req.match_id = action.match_id; 898 req.flow_key_alg = action.flow_key_alg; 899 900 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 901 } 902 903 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 904 bool enable) 905 { 906 struct npc_mcam *mcam = &rvu->hw->mcam; 907 int blkaddr, index; 908 909 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 910 if (blkaddr < 0) 911 return; 912 913 /* Get 'pcifunc' of PF device */ 914 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 915 916 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 917 NIXLF_ALLMULTI_ENTRY); 918 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 919 } 920 921 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 922 int blkaddr, u16 pcifunc, u64 rx_action) 923 { 924 int actindex, index, bank, entry; 925 struct rvu_npc_mcam_rule *rule; 926 bool enable, update; 927 928 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 929 return; 930 931 mutex_lock(&mcam->lock); 932 for (index = 0; index < mcam->bmap_entries; index++) { 933 if (mcam->entry2target_pffunc[index] == pcifunc) { 934 update = true; 935 /* update not needed for the rules added via ntuple filters */ 936 list_for_each_entry(rule, &mcam->mcam_rules, list) { 937 if (rule->entry == index) 938 update = false; 939 } 940 if (!update) 941 continue; 942 bank = npc_get_bank(mcam, index); 943 actindex = index; 944 entry = index & (mcam->banksize - 1); 945 946 /* read vf flow entry enable status */ 947 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 948 actindex); 949 /* disable before mcam entry update */ 950 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 951 false); 952 /* update 'action' */ 953 rvu_write64(rvu, blkaddr, 954 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 955 rx_action); 956 if (enable) 957 npc_enable_mcam_entry(rvu, mcam, blkaddr, 958 actindex, true); 959 } 960 } 961 mutex_unlock(&mcam->lock); 962 } 963 964 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 965 int group, int alg_idx, int mcam_index) 966 { 967 struct npc_mcam *mcam = &rvu->hw->mcam; 968 struct rvu_hwinfo *hw = rvu->hw; 969 struct nix_rx_action action; 970 int blkaddr, index, bank; 971 struct rvu_pfvf *pfvf; 972 973 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 974 if (blkaddr < 0) 975 return; 976 977 /* Check if this is for reserved default entry */ 978 if (mcam_index < 0) { 979 if (group != DEFAULT_RSS_CONTEXT_GROUP) 980 return; 981 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 982 nixlf, NIXLF_UCAST_ENTRY); 983 } else { 984 /* TODO: validate this mcam index */ 985 index = mcam_index; 986 } 987 988 if (index >= mcam->total_entries) 989 return; 990 991 bank = npc_get_bank(mcam, index); 992 index &= (mcam->banksize - 1); 993 994 *(u64 *)&action = rvu_read64(rvu, blkaddr, 995 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 996 /* Ignore if no action was set earlier */ 997 if (!*(u64 *)&action) 998 return; 999 1000 action.op = NIX_RX_ACTIONOP_RSS; 1001 action.pf_func = pcifunc; 1002 action.index = group; 1003 action.flow_key_alg = alg_idx; 1004 1005 rvu_write64(rvu, blkaddr, 1006 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 1007 1008 /* update the VF flow rule action with the VF default entry action */ 1009 if (mcam_index < 0) 1010 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1011 *(u64 *)&action); 1012 1013 /* update the action change in default rule */ 1014 pfvf = rvu_get_pfvf(rvu, pcifunc); 1015 if (pfvf->def_ucast_rule) 1016 pfvf->def_ucast_rule->rx_action = action; 1017 1018 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1019 nixlf, NIXLF_PROMISC_ENTRY); 1020 1021 /* If PF's promiscuous entry is enabled, 1022 * Set RSS action for that entry as well 1023 */ 1024 if ((!hw->cap.nix_rx_multicast || !pfvf->use_mce_list) && 1025 is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 1026 bank = npc_get_bank(mcam, index); 1027 index &= (mcam->banksize - 1); 1028 1029 rvu_write64(rvu, blkaddr, 1030 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 1031 *(u64 *)&action); 1032 } 1033 } 1034 1035 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1036 int nixlf, int type, bool enable) 1037 { 1038 struct npc_mcam *mcam = &rvu->hw->mcam; 1039 struct rvu_hwinfo *hw = rvu->hw; 1040 struct nix_mce_list *mce_list; 1041 int index, blkaddr, mce_idx; 1042 struct rvu_pfvf *pfvf; 1043 1044 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1045 if (blkaddr < 0) 1046 return; 1047 1048 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1049 nixlf, type); 1050 1051 /* disable MCAM entry when packet replication is not supported by hw */ 1052 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1053 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1054 return; 1055 } 1056 1057 /* return incase mce list is not enabled */ 1058 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1059 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1060 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1061 return; 1062 1063 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1064 1065 nix_update_mce_list(rvu, pcifunc, mce_list, 1066 mce_idx, index, enable); 1067 if (enable) 1068 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1069 } 1070 1071 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1072 int nixlf, bool enable) 1073 { 1074 struct npc_mcam *mcam = &rvu->hw->mcam; 1075 int index, blkaddr; 1076 1077 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1078 if (blkaddr < 0) 1079 return; 1080 1081 /* Ucast MCAM match entry of this PF/VF */ 1082 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1083 nixlf, NIXLF_UCAST_ENTRY); 1084 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1085 1086 /* Nothing to do for VFs, on platforms where pkt replication 1087 * is not supported 1088 */ 1089 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1090 return; 1091 1092 /* add/delete pf_func to broadcast MCE list */ 1093 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1094 NIXLF_BCAST_ENTRY, enable); 1095 } 1096 1097 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1098 { 1099 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1100 1101 /* Delete multicast and promisc MCAM entries */ 1102 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1103 NIXLF_ALLMULTI_ENTRY, false); 1104 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1105 NIXLF_PROMISC_ENTRY, false); 1106 } 1107 1108 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1109 { 1110 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1111 * in set_rx_mode mbox handler. 1112 */ 1113 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1114 } 1115 1116 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1117 { 1118 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1119 struct npc_mcam *mcam = &rvu->hw->mcam; 1120 struct rvu_npc_mcam_rule *rule, *tmp; 1121 int blkaddr; 1122 1123 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1124 if (blkaddr < 0) 1125 return; 1126 1127 mutex_lock(&mcam->lock); 1128 1129 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1130 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1131 if (is_npc_intf_rx(rule->intf) && 1132 rule->rx_action.pf_func == pcifunc && 1133 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1134 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1135 rule->entry, false); 1136 rule->enable = false; 1137 /* Indicate that default rule is disabled */ 1138 if (rule->default_rule) { 1139 pfvf->def_ucast_rule = NULL; 1140 list_del(&rule->list); 1141 kfree(rule); 1142 } 1143 } 1144 } 1145 1146 mutex_unlock(&mcam->lock); 1147 1148 npc_mcam_disable_flows(rvu, pcifunc); 1149 1150 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1151 } 1152 1153 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1154 { 1155 struct npc_mcam *mcam = &rvu->hw->mcam; 1156 struct rvu_npc_mcam_rule *rule, *tmp; 1157 int blkaddr; 1158 1159 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1160 if (blkaddr < 0) 1161 return; 1162 1163 mutex_lock(&mcam->lock); 1164 1165 /* Free all MCAM entries owned by this 'pcifunc' */ 1166 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1167 1168 /* Free all MCAM counters owned by this 'pcifunc' */ 1169 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1170 1171 /* Delete MCAM entries owned by this 'pcifunc' */ 1172 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1173 if (rule->owner == pcifunc && !rule->default_rule) { 1174 list_del(&rule->list); 1175 kfree(rule); 1176 } 1177 } 1178 1179 mutex_unlock(&mcam->lock); 1180 1181 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1182 } 1183 1184 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1185 rvu_write64(rvu, blkaddr, \ 1186 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1187 1188 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1189 rvu_write64(rvu, blkaddr, \ 1190 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1191 1192 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1193 struct npc_mcam_kex *mkex, u8 intf) 1194 { 1195 int lid, lt, ld, fl; 1196 1197 if (is_npc_intf_tx(intf)) 1198 return; 1199 1200 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1201 mkex->keyx_cfg[NIX_INTF_RX]); 1202 1203 /* Program LDATA */ 1204 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1205 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1206 for (ld = 0; ld < NPC_MAX_LD; ld++) 1207 SET_KEX_LD(intf, lid, lt, ld, 1208 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1209 [lid][lt][ld]); 1210 } 1211 } 1212 /* Program LFLAGS */ 1213 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1214 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1215 SET_KEX_LDFLAGS(intf, ld, fl, 1216 mkex->intf_ld_flags[NIX_INTF_RX] 1217 [ld][fl]); 1218 } 1219 } 1220 1221 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1222 struct npc_mcam_kex *mkex, u8 intf) 1223 { 1224 int lid, lt, ld, fl; 1225 1226 if (is_npc_intf_rx(intf)) 1227 return; 1228 1229 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1230 mkex->keyx_cfg[NIX_INTF_TX]); 1231 1232 /* Program LDATA */ 1233 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1234 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1235 for (ld = 0; ld < NPC_MAX_LD; ld++) 1236 SET_KEX_LD(intf, lid, lt, ld, 1237 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1238 [lid][lt][ld]); 1239 } 1240 } 1241 /* Program LFLAGS */ 1242 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1243 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1244 SET_KEX_LDFLAGS(intf, ld, fl, 1245 mkex->intf_ld_flags[NIX_INTF_TX] 1246 [ld][fl]); 1247 } 1248 } 1249 1250 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1251 struct npc_mcam_kex *mkex) 1252 { 1253 struct rvu_hwinfo *hw = rvu->hw; 1254 u8 intf; 1255 int ld; 1256 1257 for (ld = 0; ld < NPC_MAX_LD; ld++) 1258 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1259 mkex->kex_ld_flags[ld]); 1260 1261 for (intf = 0; intf < hw->npc_intfs; intf++) { 1262 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1263 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1264 } 1265 } 1266 1267 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1268 u64 *size) 1269 { 1270 u64 prfl_addr, prfl_sz; 1271 1272 if (!rvu->fwdata) 1273 return -EINVAL; 1274 1275 prfl_addr = rvu->fwdata->mcam_addr; 1276 prfl_sz = rvu->fwdata->mcam_sz; 1277 1278 if (!prfl_addr || !prfl_sz) 1279 return -EINVAL; 1280 1281 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1282 if (!(*prfl_img_addr)) 1283 return -ENOMEM; 1284 1285 *size = prfl_sz; 1286 1287 return 0; 1288 } 1289 1290 /* strtoull of "mkexprof" with base:36 */ 1291 #define MKEX_END_SIGN 0xdeadbeef 1292 1293 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1294 const char *mkex_profile) 1295 { 1296 struct device *dev = &rvu->pdev->dev; 1297 struct npc_mcam_kex *mcam_kex; 1298 void __iomem *mkex_prfl_addr = NULL; 1299 u64 prfl_sz; 1300 int ret; 1301 1302 /* If user not selected mkex profile */ 1303 if (rvu->kpu_fwdata_sz || 1304 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1305 goto program_mkex; 1306 1307 /* Setting up the mapping for mkex profile image */ 1308 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1309 if (ret < 0) 1310 goto program_mkex; 1311 1312 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1313 1314 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1315 /* Compare with mkex mod_param name string */ 1316 if (mcam_kex->mkex_sign == MKEX_SIGN && 1317 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1318 /* Due to an errata (35786) in A0/B0 pass silicon, 1319 * parse nibble enable configuration has to be 1320 * identical for both Rx and Tx interfaces. 1321 */ 1322 if (!is_rvu_96xx_B0(rvu) || 1323 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1324 rvu->kpu.mkex = mcam_kex; 1325 goto program_mkex; 1326 } 1327 1328 mcam_kex++; 1329 prfl_sz -= sizeof(struct npc_mcam_kex); 1330 } 1331 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1332 1333 program_mkex: 1334 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1335 /* Program selected mkex profile */ 1336 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1337 if (mkex_prfl_addr) 1338 iounmap(mkex_prfl_addr); 1339 } 1340 1341 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1342 const struct npc_kpu_profile_action *kpuaction, 1343 int kpu, int entry, bool pkind) 1344 { 1345 struct npc_kpu_action0 action0 = {0}; 1346 struct npc_kpu_action1 action1 = {0}; 1347 u64 reg; 1348 1349 action1.errlev = kpuaction->errlev; 1350 action1.errcode = kpuaction->errcode; 1351 action1.dp0_offset = kpuaction->dp0_offset; 1352 action1.dp1_offset = kpuaction->dp1_offset; 1353 action1.dp2_offset = kpuaction->dp2_offset; 1354 1355 if (pkind) 1356 reg = NPC_AF_PKINDX_ACTION1(entry); 1357 else 1358 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1359 1360 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1361 1362 action0.byp_count = kpuaction->bypass_count; 1363 action0.capture_ena = kpuaction->cap_ena; 1364 action0.parse_done = kpuaction->parse_done; 1365 action0.next_state = kpuaction->next_state; 1366 action0.capture_lid = kpuaction->lid; 1367 action0.capture_ltype = kpuaction->ltype; 1368 action0.capture_flags = kpuaction->flags; 1369 action0.ptr_advance = kpuaction->ptr_advance; 1370 action0.var_len_offset = kpuaction->offset; 1371 action0.var_len_mask = kpuaction->mask; 1372 action0.var_len_right = kpuaction->right; 1373 action0.var_len_shift = kpuaction->shift; 1374 1375 if (pkind) 1376 reg = NPC_AF_PKINDX_ACTION0(entry); 1377 else 1378 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1379 1380 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1381 } 1382 1383 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1384 const struct npc_kpu_profile_cam *kpucam, 1385 int kpu, int entry) 1386 { 1387 struct npc_kpu_cam cam0 = {0}; 1388 struct npc_kpu_cam cam1 = {0}; 1389 1390 cam1.state = kpucam->state & kpucam->state_mask; 1391 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1392 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1393 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1394 1395 cam0.state = ~kpucam->state & kpucam->state_mask; 1396 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1397 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1398 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1399 1400 rvu_write64(rvu, blkaddr, 1401 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1402 rvu_write64(rvu, blkaddr, 1403 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1404 } 1405 1406 static inline u64 enable_mask(int count) 1407 { 1408 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1409 } 1410 1411 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1412 const struct npc_kpu_profile *profile) 1413 { 1414 int entry, num_entries, max_entries; 1415 u64 entry_mask; 1416 1417 if (profile->cam_entries != profile->action_entries) { 1418 dev_err(rvu->dev, 1419 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1420 kpu, profile->cam_entries, profile->action_entries); 1421 } 1422 1423 max_entries = rvu->hw->npc_kpu_entries; 1424 1425 /* Program CAM match entries for previous KPU extracted data */ 1426 num_entries = min_t(int, profile->cam_entries, max_entries); 1427 for (entry = 0; entry < num_entries; entry++) 1428 npc_config_kpucam(rvu, blkaddr, 1429 &profile->cam[entry], kpu, entry); 1430 1431 /* Program this KPU's actions */ 1432 num_entries = min_t(int, profile->action_entries, max_entries); 1433 for (entry = 0; entry < num_entries; entry++) 1434 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1435 kpu, entry, false); 1436 1437 /* Enable all programmed entries */ 1438 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1439 entry_mask = enable_mask(num_entries); 1440 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1441 if (!rvu->kpu.custom) 1442 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1443 rvu_write64(rvu, blkaddr, 1444 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1445 if (num_entries > 64) { 1446 rvu_write64(rvu, blkaddr, 1447 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1448 enable_mask(num_entries - 64)); 1449 } 1450 1451 /* Enable this KPU */ 1452 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1453 } 1454 1455 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1456 { 1457 profile->custom = 0; 1458 profile->name = def_pfl_name; 1459 profile->version = NPC_KPU_PROFILE_VER; 1460 profile->ikpu = ikpu_action_entries; 1461 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1462 profile->kpu = npc_kpu_profiles; 1463 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1464 profile->lt_def = &npc_lt_defaults; 1465 profile->mkex = &npc_mkex_default; 1466 1467 return 0; 1468 } 1469 1470 static int npc_apply_custom_kpu(struct rvu *rvu, 1471 struct npc_kpu_profile_adapter *profile) 1472 { 1473 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1474 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1475 struct npc_kpu_profile_action *action; 1476 struct npc_kpu_profile_cam *cam; 1477 struct npc_kpu_fwdata *fw_kpu; 1478 int entries; 1479 u16 kpu, entry; 1480 1481 if (rvu->kpu_fwdata_sz < hdr_sz) { 1482 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1483 return -EINVAL; 1484 } 1485 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1486 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1487 fw->signature); 1488 return -EINVAL; 1489 } 1490 /* Verify if the using known profile structure */ 1491 if (NPC_KPU_VER_MAJ(profile->version) > 1492 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1493 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1494 NPC_KPU_VER_MAJ(profile->version), 1495 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1496 return -EINVAL; 1497 } 1498 /* Verify if profile is aligned with the required kernel changes */ 1499 if (NPC_KPU_VER_MIN(profile->version) < 1500 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1501 dev_warn(rvu->dev, 1502 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1503 NPC_KPU_VER_MAJ(profile->version), 1504 NPC_KPU_VER_MIN(profile->version), 1505 NPC_KPU_VER_PATCH(profile->version), 1506 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1507 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1508 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1509 return -EINVAL; 1510 } 1511 /* Verify if profile fits the HW */ 1512 if (fw->kpus > profile->kpus) { 1513 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1514 profile->kpus); 1515 return -EINVAL; 1516 } 1517 1518 profile->custom = 1; 1519 profile->name = fw->name; 1520 profile->version = le64_to_cpu(fw->version); 1521 profile->mkex = &fw->mkex; 1522 profile->lt_def = &fw->lt_def; 1523 1524 for (kpu = 0; kpu < fw->kpus; kpu++) { 1525 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1526 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1527 dev_warn(rvu->dev, 1528 "Too many custom entries on KPU%d: %d > %d\n", 1529 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1530 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1531 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1532 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1533 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1534 offset += fw_kpu->entries * sizeof(*action); 1535 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1536 dev_warn(rvu->dev, 1537 "Profile size mismatch on KPU%i parsing.\n", 1538 kpu + 1); 1539 return -EINVAL; 1540 } 1541 for (entry = 0; entry < entries; entry++) { 1542 profile->kpu[kpu].cam[entry] = cam[entry]; 1543 profile->kpu[kpu].action[entry] = action[entry]; 1544 } 1545 } 1546 1547 return 0; 1548 } 1549 1550 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1551 u64 prfl_sz, const char *kpu_profile) 1552 { 1553 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1554 int rc = -EINVAL; 1555 1556 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1557 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1558 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1559 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1560 kpu_profile); 1561 rvu->kpu_fwdata = kpu_data; 1562 rvu->kpu_fwdata_sz = prfl_sz; 1563 rvu->kpu_prfl_addr = prfl_addr; 1564 rc = 0; 1565 } 1566 1567 return rc; 1568 } 1569 1570 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1571 const char *kpu_profile) 1572 { 1573 struct npc_coalesced_kpu_prfl *img_data = NULL; 1574 int i = 0, rc = -EINVAL; 1575 void __iomem *kpu_prfl_addr; 1576 u16 offset; 1577 1578 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1579 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1580 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1581 /* Loaded profile is a single KPU profile. */ 1582 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1583 prfl_sz, kpu_profile); 1584 goto done; 1585 } 1586 1587 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1588 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1589 (img_data->num_prfl * sizeof(uint16_t)); 1590 /* Check if mapped image is coalesced image. */ 1591 while (i < img_data->num_prfl) { 1592 /* Profile image offsets are rounded up to next 8 multiple.*/ 1593 offset = ALIGN_8B_CEIL(offset); 1594 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1595 offset); 1596 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1597 img_data->prfl_sz[i], kpu_profile); 1598 if (!rc) 1599 break; 1600 /* Calculating offset of profile image based on profile size.*/ 1601 offset += img_data->prfl_sz[i]; 1602 i++; 1603 } 1604 done: 1605 return rc; 1606 } 1607 1608 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1609 { 1610 int ret = -EINVAL; 1611 u64 prfl_sz; 1612 1613 /* Setting up the mapping for NPC profile image */ 1614 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1615 if (ret < 0) 1616 goto done; 1617 1618 /* Detect if profile is coalesced or single KPU profile and load */ 1619 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1620 if (ret == 0) 1621 goto done; 1622 1623 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1624 if (rvu->kpu_prfl_addr) { 1625 iounmap(rvu->kpu_prfl_addr); 1626 rvu->kpu_prfl_addr = NULL; 1627 rvu->kpu_fwdata_sz = 0; 1628 rvu->kpu_fwdata = NULL; 1629 } 1630 1631 done: 1632 return ret; 1633 } 1634 1635 static void npc_load_kpu_profile(struct rvu *rvu) 1636 { 1637 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1638 const char *kpu_profile = rvu->kpu_pfl_name; 1639 const struct firmware *fw = NULL; 1640 bool retry_fwdb = false; 1641 1642 /* If user not specified profile customization */ 1643 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1644 goto revert_to_default; 1645 /* First prepare default KPU, then we'll customize top entries. */ 1646 npc_prepare_default_kpu(profile); 1647 1648 /* Order of preceedence for load loading NPC profile (high to low) 1649 * Firmware binary in filesystem. 1650 * Firmware database method. 1651 * Default KPU profile. 1652 */ 1653 if (!request_firmware(&fw, kpu_profile, rvu->dev)) { 1654 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1655 kpu_profile); 1656 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1657 if (rvu->kpu_fwdata) { 1658 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1659 rvu->kpu_fwdata_sz = fw->size; 1660 } 1661 release_firmware(fw); 1662 retry_fwdb = true; 1663 goto program_kpu; 1664 } 1665 1666 load_image_fwdb: 1667 /* Loading the KPU profile using firmware database */ 1668 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1669 goto revert_to_default; 1670 1671 program_kpu: 1672 /* Apply profile customization if firmware was loaded. */ 1673 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1674 /* If image from firmware filesystem fails to load or invalid 1675 * retry with firmware database method. 1676 */ 1677 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1678 /* Loading image from firmware database failed. */ 1679 if (rvu->kpu_prfl_addr) { 1680 iounmap(rvu->kpu_prfl_addr); 1681 rvu->kpu_prfl_addr = NULL; 1682 } else { 1683 kfree(rvu->kpu_fwdata); 1684 } 1685 rvu->kpu_fwdata = NULL; 1686 rvu->kpu_fwdata_sz = 0; 1687 if (retry_fwdb) { 1688 retry_fwdb = false; 1689 goto load_image_fwdb; 1690 } 1691 } 1692 1693 dev_warn(rvu->dev, 1694 "Can't load KPU profile %s. Using default.\n", 1695 kpu_profile); 1696 kfree(rvu->kpu_fwdata); 1697 rvu->kpu_fwdata = NULL; 1698 goto revert_to_default; 1699 } 1700 1701 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1702 profile->name, NPC_KPU_VER_MAJ(profile->version), 1703 NPC_KPU_VER_MIN(profile->version), 1704 NPC_KPU_VER_PATCH(profile->version)); 1705 1706 return; 1707 1708 revert_to_default: 1709 npc_prepare_default_kpu(profile); 1710 } 1711 1712 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1713 { 1714 struct rvu_hwinfo *hw = rvu->hw; 1715 int num_pkinds, num_kpus, idx; 1716 1717 /* Disable all KPUs and their entries */ 1718 for (idx = 0; idx < hw->npc_kpus; idx++) { 1719 rvu_write64(rvu, blkaddr, 1720 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1721 rvu_write64(rvu, blkaddr, 1722 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1723 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1724 } 1725 1726 /* Load and customize KPU profile. */ 1727 npc_load_kpu_profile(rvu); 1728 1729 /* First program IKPU profile i.e PKIND configs. 1730 * Check HW max count to avoid configuring junk or 1731 * writing to unsupported CSR addresses. 1732 */ 1733 num_pkinds = rvu->kpu.pkinds; 1734 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 1735 1736 for (idx = 0; idx < num_pkinds; idx++) 1737 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1738 1739 /* Program KPU CAM and Action profiles */ 1740 num_kpus = rvu->kpu.kpus; 1741 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1742 1743 for (idx = 0; idx < num_kpus; idx++) 1744 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1745 } 1746 1747 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1748 { 1749 int nixlf_count = rvu_get_nixlf_count(rvu); 1750 struct npc_mcam *mcam = &rvu->hw->mcam; 1751 int rsvd, err; 1752 u16 index; 1753 int cntr; 1754 u64 cfg; 1755 1756 /* Actual number of MCAM entries vary by entry size */ 1757 cfg = (rvu_read64(rvu, blkaddr, 1758 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1759 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1760 mcam->keysize = cfg; 1761 1762 /* Number of banks combined per MCAM entry */ 1763 if (cfg == NPC_MCAM_KEY_X4) 1764 mcam->banks_per_entry = 4; 1765 else if (cfg == NPC_MCAM_KEY_X2) 1766 mcam->banks_per_entry = 2; 1767 else 1768 mcam->banks_per_entry = 1; 1769 1770 /* Reserve one MCAM entry for each of the NIX LF to 1771 * guarantee space to install default matching DMAC rule. 1772 * Also reserve 2 MCAM entries for each PF for default 1773 * channel based matching or 'bcast & promisc' matching to 1774 * support BCAST and PROMISC modes of operation for PFs. 1775 * PF0 is excluded. 1776 */ 1777 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1778 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1779 if (mcam->total_entries <= rsvd) { 1780 dev_warn(rvu->dev, 1781 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1782 mcam->total_entries); 1783 return -ENOMEM; 1784 } 1785 1786 mcam->bmap_entries = mcam->total_entries - rsvd; 1787 mcam->nixlf_offset = mcam->bmap_entries; 1788 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1789 1790 /* Allocate bitmaps for managing MCAM entries */ 1791 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1792 sizeof(long), GFP_KERNEL); 1793 if (!mcam->bmap) 1794 return -ENOMEM; 1795 1796 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1797 BITS_TO_LONGS(mcam->bmap_entries), 1798 sizeof(long), GFP_KERNEL); 1799 if (!mcam->bmap_reverse) 1800 return -ENOMEM; 1801 1802 mcam->bmap_fcnt = mcam->bmap_entries; 1803 1804 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1805 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1806 sizeof(u16), GFP_KERNEL); 1807 if (!mcam->entry2pfvf_map) 1808 return -ENOMEM; 1809 1810 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1811 * allocations and another 1/8th at the top for high priority 1812 * allocations. 1813 */ 1814 mcam->lprio_count = mcam->bmap_entries / 8; 1815 if (mcam->lprio_count > BITS_PER_LONG) 1816 mcam->lprio_count = round_down(mcam->lprio_count, 1817 BITS_PER_LONG); 1818 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1819 mcam->hprio_count = mcam->lprio_count; 1820 mcam->hprio_end = mcam->hprio_count; 1821 1822 1823 /* Allocate bitmap for managing MCAM counters and memory 1824 * for saving counter to RVU PFFUNC allocation mapping. 1825 */ 1826 err = rvu_alloc_bitmap(&mcam->counters); 1827 if (err) 1828 return err; 1829 1830 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1831 sizeof(u16), GFP_KERNEL); 1832 if (!mcam->cntr2pfvf_map) 1833 goto free_mem; 1834 1835 /* Alloc memory for MCAM entry to counter mapping and for tracking 1836 * counter's reference count. 1837 */ 1838 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1839 sizeof(u16), GFP_KERNEL); 1840 if (!mcam->entry2cntr_map) 1841 goto free_mem; 1842 1843 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1844 sizeof(u16), GFP_KERNEL); 1845 if (!mcam->cntr_refcnt) 1846 goto free_mem; 1847 1848 /* Alloc memory for saving target device of mcam rule */ 1849 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1850 sizeof(u16), GFP_KERNEL); 1851 if (!mcam->entry2target_pffunc) 1852 goto free_mem; 1853 1854 for (index = 0; index < mcam->bmap_entries; index++) { 1855 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1856 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1857 } 1858 1859 for (cntr = 0; cntr < mcam->counters.max; cntr++) 1860 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1861 1862 mutex_init(&mcam->lock); 1863 1864 return 0; 1865 1866 free_mem: 1867 kfree(mcam->counters.bmap); 1868 return -ENOMEM; 1869 } 1870 1871 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1872 { 1873 struct npc_pkind *pkind = &rvu->hw->pkind; 1874 struct npc_mcam *mcam = &rvu->hw->mcam; 1875 struct rvu_hwinfo *hw = rvu->hw; 1876 u64 npc_const, npc_const1; 1877 u64 npc_const2 = 0; 1878 1879 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1880 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1881 if (npc_const1 & BIT_ULL(63)) 1882 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1883 1884 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 1885 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 1886 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1887 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1888 hw->npc_intfs = npc_const & 0xFULL; 1889 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1890 1891 mcam->banks = (npc_const >> 44) & 0xFULL; 1892 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1893 hw->npc_stat_ena = BIT_ULL(9); 1894 /* Extended set */ 1895 if (npc_const2) { 1896 hw->npc_ext_set = true; 1897 /* 96xx supports only match_stats and npc_counters 1898 * reflected in NPC_AF_CONST reg. 1899 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 1900 * 98xx has both match_stat and ext and npc_counter 1901 * reflected in NPC_AF_CONST2 1902 * STAT_SEL_EXT added at [12:14] bit position. 1903 * cn10k supports only ext and hence npc_counters in 1904 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 1905 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 1906 */ 1907 if (!hw->npc_counters) 1908 hw->npc_stat_ena = BIT_ULL(63); 1909 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1910 mcam->banksize = npc_const2 & 0xFFFFULL; 1911 } 1912 1913 mcam->counters.max = hw->npc_counters; 1914 } 1915 1916 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1917 { 1918 struct npc_mcam *mcam = &rvu->hw->mcam; 1919 struct rvu_hwinfo *hw = rvu->hw; 1920 u64 nibble_ena, rx_kex, tx_kex; 1921 u8 intf; 1922 1923 /* Reserve last counter for MCAM RX miss action which is set to 1924 * drop packet. This way we will know how many pkts didn't match 1925 * any MCAM entry. 1926 */ 1927 mcam->counters.max--; 1928 mcam->rx_miss_act_cntr = mcam->counters.max; 1929 1930 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1931 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1932 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1933 1934 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1935 if (nibble_ena) { 1936 tx_kex &= ~NPC_PARSE_NIBBLE; 1937 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1938 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1939 } 1940 1941 /* Configure RX interfaces */ 1942 for (intf = 0; intf < hw->npc_intfs; intf++) { 1943 if (is_npc_intf_tx(intf)) 1944 continue; 1945 1946 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1947 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1948 rx_kex); 1949 1950 /* If MCAM lookup doesn't result in a match, drop the received 1951 * packet. And map this action to a counter to count dropped 1952 * packets. 1953 */ 1954 rvu_write64(rvu, blkaddr, 1955 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1956 1957 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1958 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1959 */ 1960 rvu_write64(rvu, blkaddr, 1961 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1962 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1963 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 1964 } 1965 1966 /* Configure TX interfaces */ 1967 for (intf = 0; intf < hw->npc_intfs; intf++) { 1968 if (is_npc_intf_rx(intf)) 1969 continue; 1970 1971 /* Extract Ltypes LID_LA to LID_LE */ 1972 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1973 tx_kex); 1974 1975 /* Set TX miss action to UCAST_DEFAULT i.e 1976 * transmit the packet on NIX LF SQ's default channel. 1977 */ 1978 rvu_write64(rvu, blkaddr, 1979 NPC_AF_INTFX_MISS_ACT(intf), 1980 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1981 } 1982 } 1983 1984 int rvu_npc_init(struct rvu *rvu) 1985 { 1986 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1987 struct npc_pkind *pkind = &rvu->hw->pkind; 1988 struct npc_mcam *mcam = &rvu->hw->mcam; 1989 int blkaddr, entry, bank, err; 1990 1991 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1992 if (blkaddr < 0) { 1993 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1994 return -ENODEV; 1995 } 1996 1997 rvu_npc_hw_init(rvu, blkaddr); 1998 1999 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2000 for (bank = 0; bank < mcam->banks; bank++) { 2001 for (entry = 0; entry < mcam->banksize; entry++) 2002 rvu_write64(rvu, blkaddr, 2003 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2004 } 2005 2006 err = rvu_alloc_bitmap(&pkind->rsrc); 2007 if (err) 2008 return err; 2009 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2010 * no need to configure PKIND for all LBKs separately. 2011 */ 2012 rvu_alloc_rsrc(&pkind->rsrc); 2013 2014 /* Allocate mem for pkind to PF and channel mapping info */ 2015 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2016 sizeof(u32), GFP_KERNEL); 2017 if (!pkind->pfchan_map) 2018 return -ENOMEM; 2019 2020 /* Configure KPU profile */ 2021 npc_parser_profile_init(rvu, blkaddr); 2022 2023 /* Config Outer L2, IPv4's NPC layer info */ 2024 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2025 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2026 kpu->lt_def->pck_ol2.ltype_mask); 2027 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2028 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2029 kpu->lt_def->pck_oip4.ltype_mask); 2030 2031 /* Config Inner IPV4 NPC layer info */ 2032 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2033 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2034 kpu->lt_def->pck_iip4.ltype_mask); 2035 2036 /* Enable below for Rx pkts. 2037 * - Outer IPv4 header checksum validation. 2038 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2039 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2040 * - Inner IPv4 header checksum validation. 2041 * - Set non zero checksum error code value 2042 */ 2043 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2044 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2045 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2046 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2047 2048 rvu_npc_setup_interfaces(rvu, blkaddr); 2049 2050 /* Configure MKEX profile */ 2051 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2052 2053 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2054 if (err) 2055 return err; 2056 2057 err = npc_flow_steering_init(rvu, blkaddr); 2058 if (err) { 2059 dev_err(rvu->dev, 2060 "Incorrect mkex profile loaded using default mkex\n"); 2061 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2062 } 2063 2064 return 0; 2065 } 2066 2067 void rvu_npc_freemem(struct rvu *rvu) 2068 { 2069 struct npc_pkind *pkind = &rvu->hw->pkind; 2070 struct npc_mcam *mcam = &rvu->hw->mcam; 2071 2072 kfree(pkind->rsrc.bmap); 2073 kfree(mcam->counters.bmap); 2074 if (rvu->kpu_prfl_addr) 2075 iounmap(rvu->kpu_prfl_addr); 2076 else 2077 kfree(rvu->kpu_fwdata); 2078 mutex_destroy(&mcam->lock); 2079 } 2080 2081 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2082 int blkaddr, int *alloc_cnt, 2083 int *enable_cnt) 2084 { 2085 struct npc_mcam *mcam = &rvu->hw->mcam; 2086 int entry; 2087 2088 *alloc_cnt = 0; 2089 *enable_cnt = 0; 2090 2091 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2092 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2093 (*alloc_cnt)++; 2094 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2095 (*enable_cnt)++; 2096 } 2097 } 2098 } 2099 2100 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2101 int blkaddr, int *alloc_cnt, 2102 int *enable_cnt) 2103 { 2104 struct npc_mcam *mcam = &rvu->hw->mcam; 2105 int cntr; 2106 2107 *alloc_cnt = 0; 2108 *enable_cnt = 0; 2109 2110 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2111 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2112 (*alloc_cnt)++; 2113 if (mcam->cntr_refcnt[cntr]) 2114 (*enable_cnt)++; 2115 } 2116 } 2117 } 2118 2119 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 2120 u16 pcifunc, int entry) 2121 { 2122 /* verify AF installed entries */ 2123 if (is_pffunc_af(pcifunc)) 2124 return 0; 2125 /* Verify if entry is valid and if it is indeed 2126 * allocated to the requesting PFFUNC. 2127 */ 2128 if (entry >= mcam->bmap_entries) 2129 return NPC_MCAM_INVALID_REQ; 2130 2131 if (pcifunc != mcam->entry2pfvf_map[entry]) 2132 return NPC_MCAM_PERM_DENIED; 2133 2134 return 0; 2135 } 2136 2137 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2138 u16 pcifunc, int cntr) 2139 { 2140 /* Verify if counter is valid and if it is indeed 2141 * allocated to the requesting PFFUNC. 2142 */ 2143 if (cntr >= mcam->counters.max) 2144 return NPC_MCAM_INVALID_REQ; 2145 2146 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2147 return NPC_MCAM_PERM_DENIED; 2148 2149 return 0; 2150 } 2151 2152 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2153 int blkaddr, u16 entry, u16 cntr) 2154 { 2155 u16 index = entry & (mcam->banksize - 1); 2156 u32 bank = npc_get_bank(mcam, entry); 2157 struct rvu_hwinfo *hw = rvu->hw; 2158 2159 /* Set mapping and increment counter's refcnt */ 2160 mcam->entry2cntr_map[entry] = cntr; 2161 mcam->cntr_refcnt[cntr]++; 2162 /* Enable stats */ 2163 rvu_write64(rvu, blkaddr, 2164 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2165 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2166 } 2167 2168 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2169 struct npc_mcam *mcam, 2170 int blkaddr, u16 entry, u16 cntr) 2171 { 2172 u16 index = entry & (mcam->banksize - 1); 2173 u32 bank = npc_get_bank(mcam, entry); 2174 2175 /* Remove mapping and reduce counter's refcnt */ 2176 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2177 mcam->cntr_refcnt[cntr]--; 2178 /* Disable stats */ 2179 rvu_write64(rvu, blkaddr, 2180 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2181 } 2182 2183 /* Sets MCAM entry in bitmap as used. Update 2184 * reverse bitmap too. Should be called with 2185 * 'mcam->lock' held. 2186 */ 2187 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2188 { 2189 u16 entry, rentry; 2190 2191 entry = index; 2192 rentry = mcam->bmap_entries - index - 1; 2193 2194 __set_bit(entry, mcam->bmap); 2195 __set_bit(rentry, mcam->bmap_reverse); 2196 mcam->bmap_fcnt--; 2197 } 2198 2199 /* Sets MCAM entry in bitmap as free. Update 2200 * reverse bitmap too. Should be called with 2201 * 'mcam->lock' held. 2202 */ 2203 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2204 { 2205 u16 entry, rentry; 2206 2207 entry = index; 2208 rentry = mcam->bmap_entries - index - 1; 2209 2210 __clear_bit(entry, mcam->bmap); 2211 __clear_bit(rentry, mcam->bmap_reverse); 2212 mcam->bmap_fcnt++; 2213 } 2214 2215 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2216 int blkaddr, u16 pcifunc) 2217 { 2218 u16 index, cntr; 2219 2220 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2221 for (index = 0; index < mcam->bmap_entries; index++) { 2222 if (mcam->entry2pfvf_map[index] == pcifunc) { 2223 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2224 /* Free the entry in bitmap */ 2225 npc_mcam_clear_bit(mcam, index); 2226 /* Disable the entry */ 2227 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2228 2229 /* Update entry2counter mapping */ 2230 cntr = mcam->entry2cntr_map[index]; 2231 if (cntr != NPC_MCAM_INVALID_MAP) 2232 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2233 blkaddr, index, 2234 cntr); 2235 mcam->entry2target_pffunc[index] = 0x0; 2236 } 2237 } 2238 } 2239 2240 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2241 u16 pcifunc) 2242 { 2243 u16 cntr; 2244 2245 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2246 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2247 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2248 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2249 mcam->cntr_refcnt[cntr] = 0; 2250 rvu_free_rsrc(&mcam->counters, cntr); 2251 /* This API is expected to be called after freeing 2252 * MCAM entries, which inturn will remove 2253 * 'entry to counter' mapping. 2254 * No need to do it again. 2255 */ 2256 } 2257 } 2258 } 2259 2260 /* Find area of contiguous free entries of size 'nr'. 2261 * If not found return max contiguous free entries available. 2262 */ 2263 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2264 u16 nr, u16 *max_area) 2265 { 2266 u16 max_area_start = 0; 2267 u16 index, next, end; 2268 2269 *max_area = 0; 2270 2271 again: 2272 index = find_next_zero_bit(map, size, start); 2273 if (index >= size) 2274 return max_area_start; 2275 2276 end = ((index + nr) >= size) ? size : index + nr; 2277 next = find_next_bit(map, end, index); 2278 if (*max_area < (next - index)) { 2279 *max_area = next - index; 2280 max_area_start = index; 2281 } 2282 2283 if (next < end) { 2284 start = next + 1; 2285 goto again; 2286 } 2287 2288 return max_area_start; 2289 } 2290 2291 /* Find number of free MCAM entries available 2292 * within range i.e in between 'start' and 'end'. 2293 */ 2294 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2295 { 2296 u16 index, next; 2297 u16 fcnt = 0; 2298 2299 again: 2300 if (start >= end) 2301 return fcnt; 2302 2303 index = find_next_zero_bit(map, end, start); 2304 if (index >= end) 2305 return fcnt; 2306 2307 next = find_next_bit(map, end, index); 2308 if (next <= end) { 2309 fcnt += next - index; 2310 start = next + 1; 2311 goto again; 2312 } 2313 2314 fcnt += end - index; 2315 return fcnt; 2316 } 2317 2318 static void 2319 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2320 struct npc_mcam_alloc_entry_req *req, 2321 u16 *start, u16 *end, bool *reverse) 2322 { 2323 u16 fcnt; 2324 2325 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2326 goto hprio; 2327 2328 /* For a low priority entry allocation 2329 * - If reference entry is not in hprio zone then 2330 * search range: ref_entry to end. 2331 * - If reference entry is in hprio zone and if 2332 * request can be accomodated in non-hprio zone then 2333 * search range: 'start of middle zone' to 'end' 2334 * - else search in reverse, so that less number of hprio 2335 * zone entries are allocated. 2336 */ 2337 2338 *reverse = false; 2339 *start = req->ref_entry + 1; 2340 *end = mcam->bmap_entries; 2341 2342 if (req->ref_entry >= mcam->hprio_end) 2343 return; 2344 2345 fcnt = npc_mcam_get_free_count(mcam->bmap, 2346 mcam->hprio_end, mcam->bmap_entries); 2347 if (fcnt > req->count) 2348 *start = mcam->hprio_end; 2349 else 2350 *reverse = true; 2351 return; 2352 2353 hprio: 2354 /* For a high priority entry allocation, search is always 2355 * in reverse to preserve hprio zone entries. 2356 * - If reference entry is not in lprio zone then 2357 * search range: 0 to ref_entry. 2358 * - If reference entry is in lprio zone and if 2359 * request can be accomodated in middle zone then 2360 * search range: 'hprio_end' to 'lprio_start' 2361 */ 2362 2363 *reverse = true; 2364 *start = 0; 2365 *end = req->ref_entry; 2366 2367 if (req->ref_entry <= mcam->lprio_start) 2368 return; 2369 2370 fcnt = npc_mcam_get_free_count(mcam->bmap, 2371 mcam->hprio_end, mcam->lprio_start); 2372 if (fcnt < req->count) 2373 return; 2374 *start = mcam->hprio_end; 2375 *end = mcam->lprio_start; 2376 } 2377 2378 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2379 struct npc_mcam_alloc_entry_req *req, 2380 struct npc_mcam_alloc_entry_rsp *rsp) 2381 { 2382 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2383 u16 fcnt, hp_fcnt, lp_fcnt; 2384 u16 start, end, index; 2385 int entry, next_start; 2386 bool reverse = false; 2387 unsigned long *bmap; 2388 u16 max_contig; 2389 2390 mutex_lock(&mcam->lock); 2391 2392 /* Check if there are any free entries */ 2393 if (!mcam->bmap_fcnt) { 2394 mutex_unlock(&mcam->lock); 2395 return NPC_MCAM_ALLOC_FAILED; 2396 } 2397 2398 /* MCAM entries are divided into high priority, middle and 2399 * low priority zones. Idea is to not allocate top and lower 2400 * most entries as much as possible, this is to increase 2401 * probability of honouring priority allocation requests. 2402 * 2403 * Two bitmaps are used for mcam entry management, 2404 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2405 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2406 * 2407 * Reverse bitmap is used to allocate entries 2408 * - when a higher priority entry is requested 2409 * - when available free entries are less. 2410 * Lower priority ones out of avaialble free entries are always 2411 * chosen when 'high vs low' question arises. 2412 */ 2413 2414 /* Get the search range for priority allocation request */ 2415 if (req->priority) { 2416 npc_get_mcam_search_range_priority(mcam, req, 2417 &start, &end, &reverse); 2418 goto alloc; 2419 } 2420 2421 /* For a VF base MCAM match rule is set by its PF. And all the 2422 * further MCAM rules installed by VF on its own are 2423 * concatenated with the base rule set by its PF. Hence PF entries 2424 * should be at lower priority compared to VF entries. Otherwise 2425 * base rule is hit always and rules installed by VF will be of 2426 * no use. Hence if the request is from PF and NOT a priority 2427 * allocation request then allocate low priority entries. 2428 */ 2429 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 2430 goto lprio_alloc; 2431 2432 /* Find out the search range for non-priority allocation request 2433 * 2434 * Get MCAM free entry count in middle zone. 2435 */ 2436 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2437 mcam->lprio_start, 2438 mcam->bmap_entries); 2439 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2440 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2441 2442 /* Check if request can be accomodated in the middle zone */ 2443 if (fcnt > req->count) { 2444 start = mcam->hprio_end; 2445 end = mcam->lprio_start; 2446 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2447 /* Expand search zone from half of hprio zone to 2448 * half of lprio zone. 2449 */ 2450 start = mcam->hprio_end / 2; 2451 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2452 reverse = true; 2453 } else { 2454 /* Not enough free entries, search all entries in reverse, 2455 * so that low priority ones will get used up. 2456 */ 2457 lprio_alloc: 2458 reverse = true; 2459 start = 0; 2460 end = mcam->bmap_entries; 2461 } 2462 2463 alloc: 2464 if (reverse) { 2465 bmap = mcam->bmap_reverse; 2466 start = mcam->bmap_entries - start; 2467 end = mcam->bmap_entries - end; 2468 swap(start, end); 2469 } else { 2470 bmap = mcam->bmap; 2471 } 2472 2473 if (req->contig) { 2474 /* Allocate requested number of contiguous entries, if 2475 * unsuccessful find max contiguous entries available. 2476 */ 2477 index = npc_mcam_find_zero_area(bmap, end, start, 2478 req->count, &max_contig); 2479 rsp->count = max_contig; 2480 if (reverse) 2481 rsp->entry = mcam->bmap_entries - index - max_contig; 2482 else 2483 rsp->entry = index; 2484 } else { 2485 /* Allocate requested number of non-contiguous entries, 2486 * if unsuccessful allocate as many as possible. 2487 */ 2488 rsp->count = 0; 2489 next_start = start; 2490 for (entry = 0; entry < req->count; entry++) { 2491 index = find_next_zero_bit(bmap, end, next_start); 2492 if (index >= end) 2493 break; 2494 2495 next_start = start + (index - start) + 1; 2496 2497 /* Save the entry's index */ 2498 if (reverse) 2499 index = mcam->bmap_entries - index - 1; 2500 entry_list[entry] = index; 2501 rsp->count++; 2502 } 2503 } 2504 2505 /* If allocating requested no of entries is unsucessful, 2506 * expand the search range to full bitmap length and retry. 2507 */ 2508 if (!req->priority && (rsp->count < req->count) && 2509 ((end - start) != mcam->bmap_entries)) { 2510 reverse = true; 2511 start = 0; 2512 end = mcam->bmap_entries; 2513 goto alloc; 2514 } 2515 2516 /* For priority entry allocation requests, if allocation is 2517 * failed then expand search to max possible range and retry. 2518 */ 2519 if (req->priority && rsp->count < req->count) { 2520 if (req->priority == NPC_MCAM_LOWER_PRIO && 2521 (start != (req->ref_entry + 1))) { 2522 start = req->ref_entry + 1; 2523 end = mcam->bmap_entries; 2524 reverse = false; 2525 goto alloc; 2526 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2527 ((end - start) != req->ref_entry)) { 2528 start = 0; 2529 end = req->ref_entry; 2530 reverse = true; 2531 goto alloc; 2532 } 2533 } 2534 2535 /* Copy MCAM entry indices into mbox response entry_list. 2536 * Requester always expects indices in ascending order, so 2537 * so reverse the list if reverse bitmap is used for allocation. 2538 */ 2539 if (!req->contig && rsp->count) { 2540 index = 0; 2541 for (entry = rsp->count - 1; entry >= 0; entry--) { 2542 if (reverse) 2543 rsp->entry_list[index++] = entry_list[entry]; 2544 else 2545 rsp->entry_list[entry] = entry_list[entry]; 2546 } 2547 } 2548 2549 /* Mark the allocated entries as used and set nixlf mapping */ 2550 for (entry = 0; entry < rsp->count; entry++) { 2551 index = req->contig ? 2552 (rsp->entry + entry) : rsp->entry_list[entry]; 2553 npc_mcam_set_bit(mcam, index); 2554 mcam->entry2pfvf_map[index] = pcifunc; 2555 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2556 } 2557 2558 /* Update available free count in mbox response */ 2559 rsp->free_count = mcam->bmap_fcnt; 2560 2561 mutex_unlock(&mcam->lock); 2562 return 0; 2563 } 2564 2565 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2566 struct npc_mcam_alloc_entry_req *req, 2567 struct npc_mcam_alloc_entry_rsp *rsp) 2568 { 2569 struct npc_mcam *mcam = &rvu->hw->mcam; 2570 u16 pcifunc = req->hdr.pcifunc; 2571 int blkaddr; 2572 2573 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2574 if (blkaddr < 0) 2575 return NPC_MCAM_INVALID_REQ; 2576 2577 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2578 rsp->free_count = 0; 2579 2580 /* Check if ref_entry is within range */ 2581 if (req->priority && req->ref_entry >= mcam->bmap_entries) { 2582 dev_err(rvu->dev, "%s: reference entry %d is out of range\n", 2583 __func__, req->ref_entry); 2584 return NPC_MCAM_INVALID_REQ; 2585 } 2586 2587 /* ref_entry can't be '0' if requested priority is high. 2588 * Can't be last entry if requested priority is low. 2589 */ 2590 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2591 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2592 req->priority == NPC_MCAM_LOWER_PRIO)) 2593 return NPC_MCAM_INVALID_REQ; 2594 2595 /* Since list of allocated indices needs to be sent to requester, 2596 * max number of non-contiguous entries per mbox msg is limited. 2597 */ 2598 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 2599 dev_err(rvu->dev, 2600 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 2601 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 2602 return NPC_MCAM_INVALID_REQ; 2603 } 2604 2605 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2606 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2607 return NPC_MCAM_ALLOC_DENIED; 2608 2609 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2610 } 2611 2612 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2613 struct npc_mcam_free_entry_req *req, 2614 struct msg_rsp *rsp) 2615 { 2616 struct npc_mcam *mcam = &rvu->hw->mcam; 2617 u16 pcifunc = req->hdr.pcifunc; 2618 int blkaddr, rc = 0; 2619 u16 cntr; 2620 2621 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2622 if (blkaddr < 0) 2623 return NPC_MCAM_INVALID_REQ; 2624 2625 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2626 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2627 return NPC_MCAM_INVALID_REQ; 2628 2629 mutex_lock(&mcam->lock); 2630 2631 if (req->all) 2632 goto free_all; 2633 2634 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2635 if (rc) 2636 goto exit; 2637 2638 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 2639 mcam->entry2target_pffunc[req->entry] = 0x0; 2640 npc_mcam_clear_bit(mcam, req->entry); 2641 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2642 2643 /* Update entry2counter mapping */ 2644 cntr = mcam->entry2cntr_map[req->entry]; 2645 if (cntr != NPC_MCAM_INVALID_MAP) 2646 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2647 req->entry, cntr); 2648 2649 goto exit; 2650 2651 free_all: 2652 /* Free up all entries allocated to requesting PFFUNC */ 2653 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2654 exit: 2655 mutex_unlock(&mcam->lock); 2656 return rc; 2657 } 2658 2659 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2660 struct npc_mcam_read_entry_req *req, 2661 struct npc_mcam_read_entry_rsp *rsp) 2662 { 2663 struct npc_mcam *mcam = &rvu->hw->mcam; 2664 u16 pcifunc = req->hdr.pcifunc; 2665 int blkaddr, rc; 2666 2667 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2668 if (blkaddr < 0) 2669 return NPC_MCAM_INVALID_REQ; 2670 2671 mutex_lock(&mcam->lock); 2672 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2673 if (!rc) { 2674 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2675 &rsp->entry_data, 2676 &rsp->intf, &rsp->enable); 2677 } 2678 2679 mutex_unlock(&mcam->lock); 2680 return rc; 2681 } 2682 2683 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2684 struct npc_mcam_write_entry_req *req, 2685 struct msg_rsp *rsp) 2686 { 2687 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2688 struct npc_mcam *mcam = &rvu->hw->mcam; 2689 u16 pcifunc = req->hdr.pcifunc; 2690 int blkaddr, rc; 2691 u8 nix_intf; 2692 2693 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2694 if (blkaddr < 0) 2695 return NPC_MCAM_INVALID_REQ; 2696 2697 mutex_lock(&mcam->lock); 2698 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2699 if (rc) 2700 goto exit; 2701 2702 if (req->set_cntr && 2703 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2704 rc = NPC_MCAM_INVALID_REQ; 2705 goto exit; 2706 } 2707 2708 if (!is_npc_interface_valid(rvu, req->intf)) { 2709 rc = NPC_MCAM_INVALID_REQ; 2710 goto exit; 2711 } 2712 2713 if (is_npc_intf_tx(req->intf)) 2714 nix_intf = pfvf->nix_tx_intf; 2715 else 2716 nix_intf = pfvf->nix_rx_intf; 2717 2718 if (!is_pffunc_af(pcifunc) && 2719 npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, pcifunc)) { 2720 rc = NPC_MCAM_INVALID_REQ; 2721 goto exit; 2722 } 2723 2724 /* For AF installed rules, the nix_intf should be set to target NIX */ 2725 if (is_pffunc_af(req->hdr.pcifunc)) 2726 nix_intf = req->intf; 2727 2728 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2729 &req->entry_data, req->enable_entry); 2730 2731 if (req->set_cntr) 2732 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2733 req->entry, req->cntr); 2734 2735 rc = 0; 2736 exit: 2737 mutex_unlock(&mcam->lock); 2738 return rc; 2739 } 2740 2741 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2742 struct npc_mcam_ena_dis_entry_req *req, 2743 struct msg_rsp *rsp) 2744 { 2745 struct npc_mcam *mcam = &rvu->hw->mcam; 2746 u16 pcifunc = req->hdr.pcifunc; 2747 int blkaddr, rc; 2748 2749 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2750 if (blkaddr < 0) 2751 return NPC_MCAM_INVALID_REQ; 2752 2753 mutex_lock(&mcam->lock); 2754 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2755 mutex_unlock(&mcam->lock); 2756 if (rc) 2757 return rc; 2758 2759 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2760 2761 return 0; 2762 } 2763 2764 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2765 struct npc_mcam_ena_dis_entry_req *req, 2766 struct msg_rsp *rsp) 2767 { 2768 struct npc_mcam *mcam = &rvu->hw->mcam; 2769 u16 pcifunc = req->hdr.pcifunc; 2770 int blkaddr, rc; 2771 2772 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2773 if (blkaddr < 0) 2774 return NPC_MCAM_INVALID_REQ; 2775 2776 mutex_lock(&mcam->lock); 2777 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2778 mutex_unlock(&mcam->lock); 2779 if (rc) 2780 return rc; 2781 2782 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2783 2784 return 0; 2785 } 2786 2787 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2788 struct npc_mcam_shift_entry_req *req, 2789 struct npc_mcam_shift_entry_rsp *rsp) 2790 { 2791 struct npc_mcam *mcam = &rvu->hw->mcam; 2792 u16 pcifunc = req->hdr.pcifunc; 2793 u16 old_entry, new_entry; 2794 int blkaddr, rc = 0; 2795 u16 index, cntr; 2796 2797 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2798 if (blkaddr < 0) 2799 return NPC_MCAM_INVALID_REQ; 2800 2801 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2802 return NPC_MCAM_INVALID_REQ; 2803 2804 mutex_lock(&mcam->lock); 2805 for (index = 0; index < req->shift_count; index++) { 2806 old_entry = req->curr_entry[index]; 2807 new_entry = req->new_entry[index]; 2808 2809 /* Check if both old and new entries are valid and 2810 * does belong to this PFFUNC or not. 2811 */ 2812 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2813 if (rc) 2814 break; 2815 2816 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2817 if (rc) 2818 break; 2819 2820 /* new_entry should not have a counter mapped */ 2821 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2822 rc = NPC_MCAM_PERM_DENIED; 2823 break; 2824 } 2825 2826 /* Disable the new_entry */ 2827 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2828 2829 /* Copy rule from old entry to new entry */ 2830 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2831 2832 /* Copy counter mapping, if any */ 2833 cntr = mcam->entry2cntr_map[old_entry]; 2834 if (cntr != NPC_MCAM_INVALID_MAP) { 2835 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2836 old_entry, cntr); 2837 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2838 new_entry, cntr); 2839 } 2840 2841 /* Enable new_entry and disable old_entry */ 2842 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2843 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2844 } 2845 2846 /* If shift has failed then report the failed index */ 2847 if (index != req->shift_count) { 2848 rc = NPC_MCAM_PERM_DENIED; 2849 rsp->failed_entry_idx = index; 2850 } 2851 2852 mutex_unlock(&mcam->lock); 2853 return rc; 2854 } 2855 2856 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2857 struct npc_mcam_alloc_counter_req *req, 2858 struct npc_mcam_alloc_counter_rsp *rsp) 2859 { 2860 struct npc_mcam *mcam = &rvu->hw->mcam; 2861 u16 pcifunc = req->hdr.pcifunc; 2862 u16 max_contig, cntr; 2863 int blkaddr, index; 2864 2865 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2866 if (blkaddr < 0) 2867 return NPC_MCAM_INVALID_REQ; 2868 2869 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2870 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2871 return NPC_MCAM_INVALID_REQ; 2872 2873 /* Since list of allocated counter IDs needs to be sent to requester, 2874 * max number of non-contiguous counters per mbox msg is limited. 2875 */ 2876 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2877 return NPC_MCAM_INVALID_REQ; 2878 2879 mutex_lock(&mcam->lock); 2880 2881 /* Check if unused counters are available or not */ 2882 if (!rvu_rsrc_free_count(&mcam->counters)) { 2883 mutex_unlock(&mcam->lock); 2884 return NPC_MCAM_ALLOC_FAILED; 2885 } 2886 2887 rsp->count = 0; 2888 2889 if (req->contig) { 2890 /* Allocate requested number of contiguous counters, if 2891 * unsuccessful find max contiguous entries available. 2892 */ 2893 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2894 mcam->counters.max, 0, 2895 req->count, &max_contig); 2896 rsp->count = max_contig; 2897 rsp->cntr = index; 2898 for (cntr = index; cntr < (index + max_contig); cntr++) { 2899 __set_bit(cntr, mcam->counters.bmap); 2900 mcam->cntr2pfvf_map[cntr] = pcifunc; 2901 } 2902 } else { 2903 /* Allocate requested number of non-contiguous counters, 2904 * if unsuccessful allocate as many as possible. 2905 */ 2906 for (cntr = 0; cntr < req->count; cntr++) { 2907 index = rvu_alloc_rsrc(&mcam->counters); 2908 if (index < 0) 2909 break; 2910 rsp->cntr_list[cntr] = index; 2911 rsp->count++; 2912 mcam->cntr2pfvf_map[index] = pcifunc; 2913 } 2914 } 2915 2916 mutex_unlock(&mcam->lock); 2917 return 0; 2918 } 2919 2920 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2921 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2922 { 2923 struct npc_mcam *mcam = &rvu->hw->mcam; 2924 u16 index, entry = 0; 2925 int blkaddr, err; 2926 2927 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2928 if (blkaddr < 0) 2929 return NPC_MCAM_INVALID_REQ; 2930 2931 mutex_lock(&mcam->lock); 2932 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2933 if (err) { 2934 mutex_unlock(&mcam->lock); 2935 return err; 2936 } 2937 2938 /* Mark counter as free/unused */ 2939 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2940 rvu_free_rsrc(&mcam->counters, req->cntr); 2941 2942 /* Disable all MCAM entry's stats which are using this counter */ 2943 while (entry < mcam->bmap_entries) { 2944 if (!mcam->cntr_refcnt[req->cntr]) 2945 break; 2946 2947 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2948 if (index >= mcam->bmap_entries) 2949 break; 2950 entry = index + 1; 2951 if (mcam->entry2cntr_map[index] != req->cntr) 2952 continue; 2953 2954 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2955 index, req->cntr); 2956 } 2957 2958 mutex_unlock(&mcam->lock); 2959 return 0; 2960 } 2961 2962 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2963 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2964 { 2965 struct npc_mcam *mcam = &rvu->hw->mcam; 2966 u16 index, entry = 0; 2967 int blkaddr, rc; 2968 2969 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2970 if (blkaddr < 0) 2971 return NPC_MCAM_INVALID_REQ; 2972 2973 mutex_lock(&mcam->lock); 2974 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2975 if (rc) 2976 goto exit; 2977 2978 /* Unmap the MCAM entry and counter */ 2979 if (!req->all) { 2980 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2981 if (rc) 2982 goto exit; 2983 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2984 req->entry, req->cntr); 2985 goto exit; 2986 } 2987 2988 /* Disable all MCAM entry's stats which are using this counter */ 2989 while (entry < mcam->bmap_entries) { 2990 if (!mcam->cntr_refcnt[req->cntr]) 2991 break; 2992 2993 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2994 if (index >= mcam->bmap_entries) 2995 break; 2996 entry = index + 1; 2997 2998 if (mcam->entry2cntr_map[index] != req->cntr) 2999 continue; 3000 3001 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3002 index, req->cntr); 3003 } 3004 exit: 3005 mutex_unlock(&mcam->lock); 3006 return rc; 3007 } 3008 3009 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3010 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3011 { 3012 struct npc_mcam *mcam = &rvu->hw->mcam; 3013 int blkaddr, err; 3014 3015 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3016 if (blkaddr < 0) 3017 return NPC_MCAM_INVALID_REQ; 3018 3019 mutex_lock(&mcam->lock); 3020 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3021 mutex_unlock(&mcam->lock); 3022 if (err) 3023 return err; 3024 3025 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3026 3027 return 0; 3028 } 3029 3030 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 3031 struct npc_mcam_oper_counter_req *req, 3032 struct npc_mcam_oper_counter_rsp *rsp) 3033 { 3034 struct npc_mcam *mcam = &rvu->hw->mcam; 3035 int blkaddr, err; 3036 3037 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3038 if (blkaddr < 0) 3039 return NPC_MCAM_INVALID_REQ; 3040 3041 mutex_lock(&mcam->lock); 3042 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3043 mutex_unlock(&mcam->lock); 3044 if (err) 3045 return err; 3046 3047 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3048 rsp->stat &= BIT_ULL(48) - 1; 3049 3050 return 0; 3051 } 3052 3053 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3054 struct npc_mcam_alloc_and_write_entry_req *req, 3055 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3056 { 3057 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3058 struct npc_mcam_alloc_counter_req cntr_req; 3059 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3060 struct npc_mcam_alloc_entry_req entry_req; 3061 struct npc_mcam_alloc_entry_rsp entry_rsp; 3062 struct npc_mcam *mcam = &rvu->hw->mcam; 3063 u16 entry = NPC_MCAM_ENTRY_INVALID; 3064 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3065 int blkaddr, rc; 3066 u8 nix_intf; 3067 3068 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3069 if (blkaddr < 0) 3070 return NPC_MCAM_INVALID_REQ; 3071 3072 if (!is_npc_interface_valid(rvu, req->intf)) 3073 return NPC_MCAM_INVALID_REQ; 3074 3075 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 3076 req->hdr.pcifunc)) 3077 return NPC_MCAM_INVALID_REQ; 3078 3079 /* Try to allocate a MCAM entry */ 3080 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3081 entry_req.contig = true; 3082 entry_req.priority = req->priority; 3083 entry_req.ref_entry = req->ref_entry; 3084 entry_req.count = 1; 3085 3086 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3087 &entry_req, &entry_rsp); 3088 if (rc) 3089 return rc; 3090 3091 if (!entry_rsp.count) 3092 return NPC_MCAM_ALLOC_FAILED; 3093 3094 entry = entry_rsp.entry; 3095 3096 if (!req->alloc_cntr) 3097 goto write_entry; 3098 3099 /* Now allocate counter */ 3100 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3101 cntr_req.contig = true; 3102 cntr_req.count = 1; 3103 3104 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3105 if (rc) { 3106 /* Free allocated MCAM entry */ 3107 mutex_lock(&mcam->lock); 3108 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 3109 npc_mcam_clear_bit(mcam, entry); 3110 mutex_unlock(&mcam->lock); 3111 return rc; 3112 } 3113 3114 cntr = cntr_rsp.cntr; 3115 3116 write_entry: 3117 mutex_lock(&mcam->lock); 3118 3119 if (is_npc_intf_tx(req->intf)) 3120 nix_intf = pfvf->nix_tx_intf; 3121 else 3122 nix_intf = pfvf->nix_rx_intf; 3123 3124 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3125 &req->entry_data, req->enable_entry); 3126 3127 if (req->alloc_cntr) 3128 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3129 mutex_unlock(&mcam->lock); 3130 3131 rsp->entry = entry; 3132 rsp->cntr = cntr; 3133 3134 return 0; 3135 } 3136 3137 #define GET_KEX_CFG(intf) \ 3138 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3139 3140 #define GET_KEX_FLAGS(ld) \ 3141 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3142 3143 #define GET_KEX_LD(intf, lid, lt, ld) \ 3144 rvu_read64(rvu, BLKADDR_NPC, \ 3145 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3146 3147 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3148 rvu_read64(rvu, BLKADDR_NPC, \ 3149 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3150 3151 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3152 struct npc_get_kex_cfg_rsp *rsp) 3153 { 3154 int lid, lt, ld, fl; 3155 3156 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3157 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3158 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3159 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3160 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3161 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3162 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3163 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3164 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3165 } 3166 } 3167 } 3168 for (ld = 0; ld < NPC_MAX_LD; ld++) 3169 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3170 3171 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3172 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3173 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3174 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3175 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3176 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3177 } 3178 } 3179 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3180 return 0; 3181 } 3182 3183 static int 3184 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 3185 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 3186 { 3187 struct npc_kpu_action0 *act0; 3188 u8 shift_count = 0; 3189 int blkaddr; 3190 u64 val; 3191 3192 if (!var_len_off_mask) 3193 return -EINVAL; 3194 3195 if (var_len_off_mask != 0xff) { 3196 if (shift_dir) 3197 shift_count = __ffs(var_len_off_mask); 3198 else 3199 shift_count = (8 - __fls(var_len_off_mask)); 3200 } 3201 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 3202 if (blkaddr < 0) { 3203 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 3204 return -EINVAL; 3205 } 3206 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 3207 act0 = (struct npc_kpu_action0 *)&val; 3208 act0->var_len_shift = shift_count; 3209 act0->var_len_right = shift_dir; 3210 act0->var_len_mask = var_len_off_mask; 3211 act0->var_len_offset = var_len_off; 3212 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 3213 return 0; 3214 } 3215 3216 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 3217 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 3218 u8 shift_dir) 3219 3220 { 3221 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 3222 int blkaddr, nixlf, rc, intf_mode; 3223 int pf = rvu_get_pf(pcifunc); 3224 u64 rxpkind, txpkind; 3225 u8 cgx_id, lmac_id; 3226 3227 /* use default pkind to disable edsa/higig */ 3228 rxpkind = rvu_npc_get_pkind(rvu, pf); 3229 txpkind = NPC_TX_DEF_PKIND; 3230 intf_mode = NPC_INTF_MODE_DEF; 3231 3232 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 3233 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 3234 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 3235 var_len_off, 3236 var_len_off_mask, 3237 shift_dir); 3238 if (rc) 3239 return rc; 3240 } 3241 rxpkind = pkind; 3242 txpkind = pkind; 3243 } 3244 3245 if (dir & PKIND_RX) { 3246 /* rx pkind set req valid only for cgx mapped PFs */ 3247 if (!is_cgx_config_permitted(rvu, pcifunc)) 3248 return 0; 3249 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3250 3251 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 3252 rxpkind); 3253 if (rc) 3254 return rc; 3255 } 3256 3257 if (dir & PKIND_TX) { 3258 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 3259 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3260 if (rc) 3261 return rc; 3262 3263 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 3264 txpkind); 3265 } 3266 3267 pfvf->intf_mode = intf_mode; 3268 return 0; 3269 } 3270 3271 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 3272 struct msg_rsp *rsp) 3273 { 3274 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 3275 req->dir, req->pkind, req->var_len_off, 3276 req->var_len_off_mask, req->shift_dir); 3277 } 3278 3279 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3280 struct msg_req *req, 3281 struct npc_mcam_read_base_rule_rsp *rsp) 3282 { 3283 struct npc_mcam *mcam = &rvu->hw->mcam; 3284 int index, blkaddr, nixlf, rc = 0; 3285 u16 pcifunc = req->hdr.pcifunc; 3286 struct rvu_pfvf *pfvf; 3287 u8 intf, enable; 3288 3289 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3290 if (blkaddr < 0) 3291 return NPC_MCAM_INVALID_REQ; 3292 3293 /* Return the channel number in case of PF */ 3294 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3295 pfvf = rvu_get_pfvf(rvu, pcifunc); 3296 rsp->entry.kw[0] = pfvf->rx_chan_base; 3297 rsp->entry.kw_mask[0] = 0xFFFULL; 3298 goto out; 3299 } 3300 3301 /* Find the pkt steering rule installed by PF to this VF */ 3302 mutex_lock(&mcam->lock); 3303 for (index = 0; index < mcam->bmap_entries; index++) { 3304 if (mcam->entry2target_pffunc[index] == pcifunc) 3305 goto read_entry; 3306 } 3307 3308 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3309 if (rc < 0) { 3310 mutex_unlock(&mcam->lock); 3311 goto out; 3312 } 3313 /* Read the default ucast entry if there is no pkt steering rule */ 3314 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3315 NIXLF_UCAST_ENTRY); 3316 read_entry: 3317 /* Read the mcam entry */ 3318 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3319 &enable); 3320 mutex_unlock(&mcam->lock); 3321 out: 3322 return rc; 3323 } 3324 3325 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3326 struct npc_mcam_get_stats_req *req, 3327 struct npc_mcam_get_stats_rsp *rsp) 3328 { 3329 struct npc_mcam *mcam = &rvu->hw->mcam; 3330 u16 index, cntr; 3331 int blkaddr; 3332 u64 regval; 3333 u32 bank; 3334 3335 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3336 if (blkaddr < 0) 3337 return NPC_MCAM_INVALID_REQ; 3338 3339 mutex_lock(&mcam->lock); 3340 3341 index = req->entry & (mcam->banksize - 1); 3342 bank = npc_get_bank(mcam, req->entry); 3343 3344 /* read MCAM entry STAT_ACT register */ 3345 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3346 3347 if (!(regval & rvu->hw->npc_stat_ena)) { 3348 rsp->stat_ena = 0; 3349 mutex_unlock(&mcam->lock); 3350 return 0; 3351 } 3352 3353 cntr = regval & 0x1FF; 3354 3355 rsp->stat_ena = 1; 3356 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3357 rsp->stat &= BIT_ULL(48) - 1; 3358 3359 mutex_unlock(&mcam->lock); 3360 3361 return 0; 3362 } 3363