1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 */ 6 7 #include <linux/bitfield.h> 8 9 #include "rvu_struct.h" 10 #include "rvu_reg.h" 11 #include "rvu.h" 12 #include "npc.h" 13 #include "rvu_npc_fs.h" 14 #include "rvu_npc_hash.h" 15 16 #define NPC_BYTESM GENMASK_ULL(19, 16) 17 #define NPC_HDR_OFFSET GENMASK_ULL(15, 8) 18 #define NPC_KEY_OFFSET GENMASK_ULL(5, 0) 19 #define NPC_LDATA_EN BIT_ULL(7) 20 21 static const char * const npc_flow_names[] = { 22 [NPC_DMAC] = "dmac", 23 [NPC_SMAC] = "smac", 24 [NPC_ETYPE] = "ether type", 25 [NPC_VLAN_ETYPE_CTAG] = "vlan ether type ctag", 26 [NPC_VLAN_ETYPE_STAG] = "vlan ether type stag", 27 [NPC_OUTER_VID] = "outer vlan id", 28 [NPC_TOS] = "tos", 29 [NPC_SIP_IPV4] = "ipv4 source ip", 30 [NPC_DIP_IPV4] = "ipv4 destination ip", 31 [NPC_SIP_IPV6] = "ipv6 source ip", 32 [NPC_DIP_IPV6] = "ipv6 destination ip", 33 [NPC_IPPROTO_TCP] = "ip proto tcp", 34 [NPC_IPPROTO_UDP] = "ip proto udp", 35 [NPC_IPPROTO_SCTP] = "ip proto sctp", 36 [NPC_IPPROTO_ICMP] = "ip proto icmp", 37 [NPC_IPPROTO_ICMP6] = "ip proto icmp6", 38 [NPC_IPPROTO_AH] = "ip proto AH", 39 [NPC_IPPROTO_ESP] = "ip proto ESP", 40 [NPC_SPORT_TCP] = "tcp source port", 41 [NPC_DPORT_TCP] = "tcp destination port", 42 [NPC_SPORT_UDP] = "udp source port", 43 [NPC_DPORT_UDP] = "udp destination port", 44 [NPC_SPORT_SCTP] = "sctp source port", 45 [NPC_DPORT_SCTP] = "sctp destination port", 46 [NPC_UNKNOWN] = "unknown", 47 }; 48 49 const char *npc_get_field_name(u8 hdr) 50 { 51 if (hdr >= ARRAY_SIZE(npc_flow_names)) 52 return npc_flow_names[NPC_UNKNOWN]; 53 54 return npc_flow_names[hdr]; 55 } 56 57 /* Compute keyword masks and figure out the number of keywords a field 58 * spans in the key. 59 */ 60 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, 61 u8 nr_bits, int start_kwi, int offset, u8 intf) 62 { 63 struct npc_key_field *field = &mcam->rx_key_fields[type]; 64 u8 bits_in_kw; 65 int max_kwi; 66 67 if (mcam->banks_per_entry == 1) 68 max_kwi = 1; /* NPC_MCAM_KEY_X1 */ 69 else if (mcam->banks_per_entry == 2) 70 max_kwi = 3; /* NPC_MCAM_KEY_X2 */ 71 else 72 max_kwi = 6; /* NPC_MCAM_KEY_X4 */ 73 74 if (is_npc_intf_tx(intf)) 75 field = &mcam->tx_key_fields[type]; 76 77 if (offset + nr_bits <= 64) { 78 /* one KW only */ 79 if (start_kwi > max_kwi) 80 return; 81 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0) 82 << offset; 83 field->nr_kws = 1; 84 } else if (offset + nr_bits > 64 && 85 offset + nr_bits <= 128) { 86 /* two KWs */ 87 if (start_kwi + 1 > max_kwi) 88 return; 89 /* first KW mask */ 90 bits_in_kw = 64 - offset; 91 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 92 << offset; 93 /* second KW mask i.e. mask for rest of bits */ 94 bits_in_kw = nr_bits + offset - 64; 95 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0); 96 field->nr_kws = 2; 97 } else { 98 /* three KWs */ 99 if (start_kwi + 2 > max_kwi) 100 return; 101 /* first KW mask */ 102 bits_in_kw = 64 - offset; 103 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 104 << offset; 105 /* second KW mask */ 106 field->kw_mask[start_kwi + 1] = ~0ULL; 107 /* third KW mask i.e. mask for rest of bits */ 108 bits_in_kw = nr_bits + offset - 128; 109 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0); 110 field->nr_kws = 3; 111 } 112 } 113 114 /* Helper function to figure out whether field exists in the key */ 115 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf) 116 { 117 struct npc_mcam *mcam = &rvu->hw->mcam; 118 struct npc_key_field *input; 119 120 input = &mcam->rx_key_fields[type]; 121 if (is_npc_intf_tx(intf)) 122 input = &mcam->tx_key_fields[type]; 123 124 return input->nr_kws > 0; 125 } 126 127 static bool npc_is_same(struct npc_key_field *input, 128 struct npc_key_field *field) 129 { 130 return memcmp(&input->layer_mdata, &field->layer_mdata, 131 sizeof(struct npc_layer_mdata)) == 0; 132 } 133 134 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, 135 u64 cfg, u8 lid, u8 lt, u8 intf) 136 { 137 struct npc_key_field *input = &mcam->rx_key_fields[type]; 138 139 if (is_npc_intf_tx(intf)) 140 input = &mcam->tx_key_fields[type]; 141 142 input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 143 input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); 144 input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; 145 input->layer_mdata.ltype = lt; 146 input->layer_mdata.lid = lid; 147 } 148 149 static bool npc_check_overlap_fields(struct npc_key_field *input1, 150 struct npc_key_field *input2) 151 { 152 int kwi; 153 154 /* Fields with same layer id and different ltypes are mutually 155 * exclusive hence they can be overlapped 156 */ 157 if (input1->layer_mdata.lid == input2->layer_mdata.lid && 158 input1->layer_mdata.ltype != input2->layer_mdata.ltype) 159 return false; 160 161 for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { 162 if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) 163 return true; 164 } 165 166 return false; 167 } 168 169 /* Helper function to check whether given field overlaps with any other fields 170 * in the key. Due to limitations on key size and the key extraction profile in 171 * use higher layers can overwrite lower layer's header fields. Hence overlap 172 * needs to be checked. 173 */ 174 static bool npc_check_overlap(struct rvu *rvu, int blkaddr, 175 enum key_fields type, u8 start_lid, u8 intf) 176 { 177 struct npc_mcam *mcam = &rvu->hw->mcam; 178 struct npc_key_field *dummy, *input; 179 int start_kwi, offset; 180 u8 nr_bits, lid, lt, ld; 181 u64 cfg; 182 183 dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; 184 input = &mcam->rx_key_fields[type]; 185 186 if (is_npc_intf_tx(intf)) { 187 dummy = &mcam->tx_key_fields[NPC_UNKNOWN]; 188 input = &mcam->tx_key_fields[type]; 189 } 190 191 for (lid = start_lid; lid < NPC_MAX_LID; lid++) { 192 for (lt = 0; lt < NPC_MAX_LT; lt++) { 193 for (ld = 0; ld < NPC_MAX_LD; ld++) { 194 cfg = rvu_read64(rvu, blkaddr, 195 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 196 (intf, lid, lt, ld)); 197 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 198 continue; 199 memset(dummy, 0, sizeof(struct npc_key_field)); 200 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, 201 lid, lt, intf); 202 /* exclude input */ 203 if (npc_is_same(input, dummy)) 204 continue; 205 start_kwi = dummy->layer_mdata.key / 8; 206 offset = (dummy->layer_mdata.key * 8) % 64; 207 nr_bits = dummy->layer_mdata.len * 8; 208 /* form KW masks */ 209 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, 210 start_kwi, offset, intf); 211 /* check any input field bits falls in any 212 * other field bits. 213 */ 214 if (npc_check_overlap_fields(dummy, input)) 215 return true; 216 } 217 } 218 } 219 220 return false; 221 } 222 223 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, 224 u8 intf) 225 { 226 if (!npc_is_field_present(rvu, type, intf) || 227 npc_check_overlap(rvu, blkaddr, type, 0, intf)) 228 return false; 229 return true; 230 } 231 232 static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, 233 u8 key_nibble, u8 intf) 234 { 235 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 236 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 237 u8 nr_bits = 4; /* bits in a nibble */ 238 u8 type; 239 240 switch (bit_number) { 241 case 40 ... 43: 242 type = NPC_EXACT_RESULT; 243 break; 244 245 default: 246 return; 247 } 248 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 249 } 250 251 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, 252 u8 key_nibble, u8 intf) 253 { 254 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 255 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 256 u8 nr_bits = 4; /* bits in a nibble */ 257 u8 type; 258 259 switch (bit_number) { 260 case 0 ... 2: 261 type = NPC_CHAN; 262 break; 263 case 3: 264 type = NPC_ERRLEV; 265 break; 266 case 4 ... 5: 267 type = NPC_ERRCODE; 268 break; 269 case 6: 270 type = NPC_LXMB; 271 break; 272 /* check for LTYPE only as of now */ 273 case 9: 274 type = NPC_LA; 275 break; 276 case 12: 277 type = NPC_LB; 278 break; 279 case 15: 280 type = NPC_LC; 281 break; 282 case 18: 283 type = NPC_LD; 284 break; 285 case 21: 286 type = NPC_LE; 287 break; 288 case 24: 289 type = NPC_LF; 290 break; 291 case 27: 292 type = NPC_LG; 293 break; 294 case 30: 295 type = NPC_LH; 296 break; 297 default: 298 return; 299 } 300 301 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 302 } 303 304 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) 305 { 306 struct npc_mcam *mcam = &rvu->hw->mcam; 307 struct npc_key_field *key_fields; 308 /* Ether type can come from three layers 309 * (ethernet, single tagged, double tagged) 310 */ 311 struct npc_key_field *etype_ether; 312 struct npc_key_field *etype_tag1; 313 struct npc_key_field *etype_tag2; 314 /* Outer VLAN TCI can come from two layers 315 * (single tagged, double tagged) 316 */ 317 struct npc_key_field *vlan_tag1; 318 struct npc_key_field *vlan_tag2; 319 u64 *features; 320 u8 start_lid; 321 int i; 322 323 key_fields = mcam->rx_key_fields; 324 features = &mcam->rx_features; 325 326 if (is_npc_intf_tx(intf)) { 327 key_fields = mcam->tx_key_fields; 328 features = &mcam->tx_features; 329 } 330 331 /* Handle header fields which can come from multiple layers like 332 * etype, outer vlan tci. These fields should have same position in 333 * the key otherwise to install a mcam rule more than one entry is 334 * needed which complicates mcam space management. 335 */ 336 etype_ether = &key_fields[NPC_ETYPE_ETHER]; 337 etype_tag1 = &key_fields[NPC_ETYPE_TAG1]; 338 etype_tag2 = &key_fields[NPC_ETYPE_TAG2]; 339 vlan_tag1 = &key_fields[NPC_VLAN_TAG1]; 340 vlan_tag2 = &key_fields[NPC_VLAN_TAG2]; 341 342 /* if key profile programmed does not extract Ethertype at all */ 343 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 344 goto vlan_tci; 345 346 /* if key profile programmed extracts Ethertype from one layer */ 347 if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 348 key_fields[NPC_ETYPE] = *etype_ether; 349 if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws) 350 key_fields[NPC_ETYPE] = *etype_tag1; 351 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws) 352 key_fields[NPC_ETYPE] = *etype_tag2; 353 354 /* if key profile programmed extracts Ethertype from multiple layers */ 355 if (etype_ether->nr_kws && etype_tag1->nr_kws) { 356 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 357 if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) 358 goto vlan_tci; 359 } 360 key_fields[NPC_ETYPE] = *etype_tag1; 361 } 362 if (etype_ether->nr_kws && etype_tag2->nr_kws) { 363 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 364 if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) 365 goto vlan_tci; 366 } 367 key_fields[NPC_ETYPE] = *etype_tag2; 368 } 369 if (etype_tag1->nr_kws && etype_tag2->nr_kws) { 370 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 371 if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) 372 goto vlan_tci; 373 } 374 key_fields[NPC_ETYPE] = *etype_tag2; 375 } 376 377 /* check none of higher layers overwrite Ethertype */ 378 start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1; 379 if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf)) 380 goto vlan_tci; 381 *features |= BIT_ULL(NPC_ETYPE); 382 vlan_tci: 383 /* if key profile does not extract outer vlan tci at all */ 384 if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 385 goto done; 386 387 /* if key profile extracts outer vlan tci from one layer */ 388 if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 389 key_fields[NPC_OUTER_VID] = *vlan_tag1; 390 if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws) 391 key_fields[NPC_OUTER_VID] = *vlan_tag2; 392 393 /* if key profile extracts outer vlan tci from multiple layers */ 394 if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { 395 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 396 if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) 397 goto done; 398 } 399 key_fields[NPC_OUTER_VID] = *vlan_tag2; 400 } 401 /* check none of higher layers overwrite outer vlan tci */ 402 start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1; 403 if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf)) 404 goto done; 405 *features |= BIT_ULL(NPC_OUTER_VID); 406 done: 407 return; 408 } 409 410 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, 411 u8 lt, u64 cfg, u8 intf) 412 { 413 struct npc_mcam *mcam = &rvu->hw->mcam; 414 u8 hdr, key, nr_bytes, bit_offset; 415 u8 la_ltype, la_start; 416 /* starting KW index and starting bit position */ 417 int start_kwi, offset; 418 419 nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; 420 hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 421 key = FIELD_GET(NPC_KEY_OFFSET, cfg); 422 start_kwi = key / 8; 423 offset = (key * 8) % 64; 424 425 /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding 426 * ethernet header. 427 */ 428 if (is_npc_intf_tx(intf)) { 429 la_ltype = NPC_LT_LA_IH_NIX_ETHER; 430 la_start = 8; 431 } else { 432 la_ltype = NPC_LT_LA_ETHER; 433 la_start = 0; 434 } 435 436 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen) \ 437 do { \ 438 if (lid == (hlid) && lt == (hlt)) { \ 439 if ((hstart) >= hdr && \ 440 ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ 441 bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ 442 npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ 443 npc_set_kw_masks(mcam, (name), (hlen) * 8, \ 444 start_kwi, offset + bit_offset, intf);\ 445 } \ 446 } \ 447 } while (0) 448 449 /* List LID, LTYPE, start offset from layer and length(in bytes) of 450 * packet header fields below. 451 * Example: Source IP is 4 bytes and starts at 12th byte of IP header 452 */ 453 NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1); 454 NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4); 455 NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4); 456 NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16); 457 NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16); 458 NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2); 459 NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2); 460 NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2); 461 NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2); 462 NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2); 463 NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2); 464 NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2); 465 NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2); 466 NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2); 467 NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); 468 NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); 469 NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); 470 /* SMAC follows the DMAC(which is 6 bytes) */ 471 NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start + 6, 6); 472 /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ 473 NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2); 474 } 475 476 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) 477 { 478 struct npc_mcam *mcam = &rvu->hw->mcam; 479 u64 *features = &mcam->rx_features; 480 u64 tcp_udp_sctp; 481 int hdr; 482 483 if (is_npc_intf_tx(intf)) 484 features = &mcam->tx_features; 485 486 for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) { 487 if (npc_check_field(rvu, blkaddr, hdr, intf)) 488 *features |= BIT_ULL(hdr); 489 } 490 491 tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) | 492 BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) | 493 BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP); 494 495 /* for tcp/udp/sctp corresponding layer type should be in the key */ 496 if (*features & tcp_udp_sctp) { 497 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf)) 498 *features &= ~tcp_udp_sctp; 499 else 500 *features |= BIT_ULL(NPC_IPPROTO_TCP) | 501 BIT_ULL(NPC_IPPROTO_UDP) | 502 BIT_ULL(NPC_IPPROTO_SCTP); 503 } 504 505 /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */ 506 if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) { 507 *features |= BIT_ULL(NPC_IPPROTO_AH); 508 *features |= BIT_ULL(NPC_IPPROTO_ICMP); 509 *features |= BIT_ULL(NPC_IPPROTO_ICMP6); 510 } 511 512 /* for ESP, check if corresponding layer type is present in the key */ 513 if (npc_check_field(rvu, blkaddr, NPC_LE, intf)) 514 *features |= BIT_ULL(NPC_IPPROTO_ESP); 515 516 /* for vlan corresponding layer type should be in the key */ 517 if (*features & BIT_ULL(NPC_OUTER_VID)) 518 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) 519 *features &= ~BIT_ULL(NPC_OUTER_VID); 520 521 /* for vlan ethertypes corresponding layer type should be in the key */ 522 if (npc_check_field(rvu, blkaddr, NPC_LB, intf)) 523 *features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) | 524 BIT_ULL(NPC_VLAN_ETYPE_STAG); 525 } 526 527 /* Scan key extraction profile and record how fields of our interest 528 * fill the key structure. Also verify Channel and DMAC exists in 529 * key and not overwritten by other header fields. 530 */ 531 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) 532 { 533 struct npc_mcam *mcam = &rvu->hw->mcam; 534 u8 lid, lt, ld, bitnr; 535 u64 cfg, masked_cfg; 536 u8 key_nibble = 0; 537 538 /* Scan and note how parse result is going to be in key. 539 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from 540 * parse result in the key. The enabled nibbles from parse result 541 * will be concatenated in key. 542 */ 543 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 544 masked_cfg = cfg & NPC_PARSE_NIBBLE; 545 for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) { 546 npc_scan_parse_result(mcam, bitnr, key_nibble, intf); 547 key_nibble++; 548 } 549 550 /* Ignore exact match bits for mcam entries except the first rule 551 * which is drop on hit. This first rule is configured explitcitly by 552 * exact match code. 553 */ 554 masked_cfg = cfg & NPC_EXACT_NIBBLE; 555 bitnr = NPC_EXACT_NIBBLE_START; 556 for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, 557 NPC_EXACT_NIBBLE_START) { 558 npc_scan_exact_result(mcam, bitnr, key_nibble, intf); 559 key_nibble++; 560 } 561 562 /* Scan and note how layer data is going to be in key */ 563 for (lid = 0; lid < NPC_MAX_LID; lid++) { 564 for (lt = 0; lt < NPC_MAX_LT; lt++) { 565 for (ld = 0; ld < NPC_MAX_LD; ld++) { 566 cfg = rvu_read64(rvu, blkaddr, 567 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 568 (intf, lid, lt, ld)); 569 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 570 continue; 571 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, 572 intf); 573 } 574 } 575 } 576 577 return 0; 578 } 579 580 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) 581 { 582 int err; 583 584 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX); 585 if (err) 586 return err; 587 588 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX); 589 if (err) 590 return err; 591 592 /* Channel is mandatory */ 593 if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) { 594 dev_err(rvu->dev, "Channel not present in Key\n"); 595 return -EINVAL; 596 } 597 /* check that none of the fields overwrite channel */ 598 if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) { 599 dev_err(rvu->dev, "Channel cannot be overwritten\n"); 600 return -EINVAL; 601 } 602 /* DMAC should be present in key for unicast filter to work */ 603 if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) { 604 dev_err(rvu->dev, "DMAC not present in Key\n"); 605 return -EINVAL; 606 } 607 /* check that none of the fields overwrite DMAC */ 608 if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) { 609 dev_err(rvu->dev, "DMAC cannot be overwritten\n"); 610 return -EINVAL; 611 } 612 613 npc_set_features(rvu, blkaddr, NIX_INTF_TX); 614 npc_set_features(rvu, blkaddr, NIX_INTF_RX); 615 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX); 616 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX); 617 618 return 0; 619 } 620 621 int npc_flow_steering_init(struct rvu *rvu, int blkaddr) 622 { 623 struct npc_mcam *mcam = &rvu->hw->mcam; 624 625 INIT_LIST_HEAD(&mcam->mcam_rules); 626 627 return npc_scan_verify_kex(rvu, blkaddr); 628 } 629 630 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) 631 { 632 struct npc_mcam *mcam = &rvu->hw->mcam; 633 u64 *mcam_features = &mcam->rx_features; 634 u64 unsupported; 635 u8 bit; 636 637 if (is_npc_intf_tx(intf)) 638 mcam_features = &mcam->tx_features; 639 640 unsupported = (*mcam_features ^ features) & ~(*mcam_features); 641 if (unsupported) { 642 dev_info(rvu->dev, "Unsupported flow(s):\n"); 643 for_each_set_bit(bit, (unsigned long *)&unsupported, 64) 644 dev_info(rvu->dev, "%s ", npc_get_field_name(bit)); 645 return -EOPNOTSUPP; 646 } 647 648 return 0; 649 } 650 651 /* npc_update_entry - Based on the masks generated during 652 * the key scanning, updates the given entry with value and 653 * masks for the field of interest. Maximum 16 bytes of a packet 654 * header can be extracted by HW hence lo and hi are sufficient. 655 * When field bytes are less than or equal to 8 then hi should be 656 * 0 for value and mask. 657 * 658 * If exact match of value is required then mask should be all 1's. 659 * If any bits in mask are 0 then corresponding bits in value are 660 * dont care. 661 */ 662 void npc_update_entry(struct rvu *rvu, enum key_fields type, 663 struct mcam_entry *entry, u64 val_lo, 664 u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) 665 { 666 struct npc_mcam *mcam = &rvu->hw->mcam; 667 struct mcam_entry dummy = { {0} }; 668 struct npc_key_field *field; 669 u64 kw1, kw2, kw3; 670 u8 shift; 671 int i; 672 673 field = &mcam->rx_key_fields[type]; 674 if (is_npc_intf_tx(intf)) 675 field = &mcam->tx_key_fields[type]; 676 677 if (!field->nr_kws) 678 return; 679 680 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 681 if (!field->kw_mask[i]) 682 continue; 683 /* place key value in kw[x] */ 684 shift = __ffs64(field->kw_mask[i]); 685 /* update entry value */ 686 kw1 = (val_lo << shift) & field->kw_mask[i]; 687 dummy.kw[i] = kw1; 688 /* update entry mask */ 689 kw1 = (mask_lo << shift) & field->kw_mask[i]; 690 dummy.kw_mask[i] = kw1; 691 692 if (field->nr_kws == 1) 693 break; 694 /* place remaining bits of key value in kw[x + 1] */ 695 if (field->nr_kws == 2) { 696 /* update entry value */ 697 kw2 = shift ? val_lo >> (64 - shift) : 0; 698 kw2 |= (val_hi << shift); 699 kw2 &= field->kw_mask[i + 1]; 700 dummy.kw[i + 1] = kw2; 701 /* update entry mask */ 702 kw2 = shift ? mask_lo >> (64 - shift) : 0; 703 kw2 |= (mask_hi << shift); 704 kw2 &= field->kw_mask[i + 1]; 705 dummy.kw_mask[i + 1] = kw2; 706 break; 707 } 708 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ 709 if (field->nr_kws == 3) { 710 /* update entry value */ 711 kw2 = shift ? val_lo >> (64 - shift) : 0; 712 kw2 |= (val_hi << shift); 713 kw2 &= field->kw_mask[i + 1]; 714 kw3 = shift ? val_hi >> (64 - shift) : 0; 715 kw3 &= field->kw_mask[i + 2]; 716 dummy.kw[i + 1] = kw2; 717 dummy.kw[i + 2] = kw3; 718 /* update entry mask */ 719 kw2 = shift ? mask_lo >> (64 - shift) : 0; 720 kw2 |= (mask_hi << shift); 721 kw2 &= field->kw_mask[i + 1]; 722 kw3 = shift ? mask_hi >> (64 - shift) : 0; 723 kw3 &= field->kw_mask[i + 2]; 724 dummy.kw_mask[i + 1] = kw2; 725 dummy.kw_mask[i + 2] = kw3; 726 break; 727 } 728 } 729 /* dummy is ready with values and masks for given key 730 * field now clear and update input entry with those 731 */ 732 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 733 if (!field->kw_mask[i]) 734 continue; 735 entry->kw[i] &= ~field->kw_mask[i]; 736 entry->kw_mask[i] &= ~field->kw_mask[i]; 737 738 entry->kw[i] |= dummy.kw[i]; 739 entry->kw_mask[i] |= dummy.kw_mask[i]; 740 } 741 } 742 743 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, 744 u64 features, struct flow_msg *pkt, 745 struct flow_msg *mask, 746 struct rvu_npc_mcam_rule *output, u8 intf) 747 { 748 u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS]; 749 u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS]; 750 struct flow_msg *opkt = &output->packet; 751 struct flow_msg *omask = &output->mask; 752 u64 mask_lo, mask_hi; 753 u64 val_lo, val_hi; 754 755 /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet 756 * values to be programmed in MCAM should as below: 757 * val_high: 0xfe80000000000000 758 * val_low: 0x2c6863fffe5e2d0a 759 */ 760 if (features & BIT_ULL(NPC_SIP_IPV6)) { 761 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS); 762 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); 763 764 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1]; 765 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3]; 766 val_hi = (u64)src_ip[0] << 32 | src_ip[1]; 767 val_lo = (u64)src_ip[2] << 32 | src_ip[3]; 768 769 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, 770 mask_lo, mask_hi, intf); 771 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); 772 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); 773 } 774 if (features & BIT_ULL(NPC_DIP_IPV6)) { 775 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS); 776 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); 777 778 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1]; 779 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3]; 780 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; 781 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; 782 783 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, 784 mask_lo, mask_hi, intf); 785 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); 786 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); 787 } 788 } 789 790 static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, 791 u64 features, u8 intf) 792 { 793 bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG)); 794 bool stag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_STAG)); 795 bool vid = !!(features & BIT_ULL(NPC_OUTER_VID)); 796 797 /* If only VLAN id is given then always match outer VLAN id */ 798 if (vid && !ctag && !stag) { 799 npc_update_entry(rvu, NPC_LB, entry, 800 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, 801 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); 802 return; 803 } 804 if (ctag) 805 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0, 806 ~0ULL, 0, intf); 807 if (stag) 808 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0, 809 ~0ULL, 0, intf); 810 } 811 812 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, 813 u64 features, struct flow_msg *pkt, 814 struct flow_msg *mask, 815 struct rvu_npc_mcam_rule *output, u8 intf, 816 int blkaddr) 817 { 818 u64 dmac_mask = ether_addr_to_u64(mask->dmac); 819 u64 smac_mask = ether_addr_to_u64(mask->smac); 820 u64 dmac_val = ether_addr_to_u64(pkt->dmac); 821 u64 smac_val = ether_addr_to_u64(pkt->smac); 822 struct flow_msg *opkt = &output->packet; 823 struct flow_msg *omask = &output->mask; 824 825 if (!features) 826 return; 827 828 /* For tcp/udp/sctp LTYPE should be present in entry */ 829 if (features & BIT_ULL(NPC_IPPROTO_TCP)) 830 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, 831 0, ~0ULL, 0, intf); 832 if (features & BIT_ULL(NPC_IPPROTO_UDP)) 833 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, 834 0, ~0ULL, 0, intf); 835 if (features & BIT_ULL(NPC_IPPROTO_SCTP)) 836 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, 837 0, ~0ULL, 0, intf); 838 if (features & BIT_ULL(NPC_IPPROTO_ICMP)) 839 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, 840 0, ~0ULL, 0, intf); 841 if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) 842 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, 843 0, ~0ULL, 0, intf); 844 845 /* For AH, LTYPE should be present in entry */ 846 if (features & BIT_ULL(NPC_IPPROTO_AH)) 847 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, 848 0, ~0ULL, 0, intf); 849 /* For ESP, LTYPE should be present in entry */ 850 if (features & BIT_ULL(NPC_IPPROTO_ESP)) 851 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, 852 0, ~0ULL, 0, intf); 853 854 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ 855 do { \ 856 if (features & BIT_ULL((field))) { \ 857 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ 858 (mask_lo), (mask_hi), intf); \ 859 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ 860 memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ 861 } \ 862 } while (0) 863 864 NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0); 865 866 NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0); 867 NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0, 868 ntohs(mask->etype), 0); 869 NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0); 870 NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0, 871 ntohl(mask->ip4src), 0); 872 NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0, 873 ntohl(mask->ip4dst), 0); 874 NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0, 875 ntohs(mask->sport), 0); 876 NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0, 877 ntohs(mask->sport), 0); 878 NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0, 879 ntohs(mask->dport), 0); 880 NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0, 881 ntohs(mask->dport), 0); 882 NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0, 883 ntohs(mask->sport), 0); 884 NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0, 885 ntohs(mask->dport), 0); 886 887 NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0, 888 ntohs(mask->vlan_tci), 0); 889 890 npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); 891 npc_update_vlan_features(rvu, entry, features, intf); 892 893 npc_update_field_hash(rvu, intf, entry, blkaddr, features, 894 pkt, mask, opkt, omask); 895 } 896 897 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, u16 entry) 898 { 899 struct rvu_npc_mcam_rule *iter; 900 901 mutex_lock(&mcam->lock); 902 list_for_each_entry(iter, &mcam->mcam_rules, list) { 903 if (iter->entry == entry) { 904 mutex_unlock(&mcam->lock); 905 return iter; 906 } 907 } 908 mutex_unlock(&mcam->lock); 909 910 return NULL; 911 } 912 913 static void rvu_mcam_add_rule(struct npc_mcam *mcam, 914 struct rvu_npc_mcam_rule *rule) 915 { 916 struct list_head *head = &mcam->mcam_rules; 917 struct rvu_npc_mcam_rule *iter; 918 919 mutex_lock(&mcam->lock); 920 list_for_each_entry(iter, &mcam->mcam_rules, list) { 921 if (iter->entry > rule->entry) 922 break; 923 head = &iter->list; 924 } 925 926 list_add(&rule->list, head); 927 mutex_unlock(&mcam->lock); 928 } 929 930 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 931 struct rvu_npc_mcam_rule *rule) 932 { 933 struct npc_mcam_oper_counter_req free_req = { 0 }; 934 struct msg_rsp free_rsp; 935 936 if (!rule->has_cntr) 937 return; 938 939 free_req.hdr.pcifunc = pcifunc; 940 free_req.cntr = rule->cntr; 941 942 rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp); 943 rule->has_cntr = false; 944 } 945 946 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 947 struct rvu_npc_mcam_rule *rule, 948 struct npc_install_flow_rsp *rsp) 949 { 950 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 951 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 952 int err; 953 954 cntr_req.hdr.pcifunc = pcifunc; 955 cntr_req.contig = true; 956 cntr_req.count = 1; 957 958 /* we try to allocate a counter to track the stats of this 959 * rule. If counter could not be allocated then proceed 960 * without counter because counters are limited than entries. 961 */ 962 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, 963 &cntr_rsp); 964 if (!err && cntr_rsp.count) { 965 rule->cntr = cntr_rsp.cntr; 966 rule->has_cntr = true; 967 rsp->counter = rule->cntr; 968 } else { 969 rsp->counter = err; 970 } 971 } 972 973 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 974 struct mcam_entry *entry, 975 struct npc_install_flow_req *req, 976 u16 target, bool pf_set_vfs_mac) 977 { 978 struct rvu_switch *rswitch = &rvu->rswitch; 979 struct nix_rx_action action; 980 981 if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) 982 req->chan_mask = 0x0; /* Do not care channel */ 983 984 npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, 985 0, NIX_INTF_RX); 986 987 *(u64 *)&action = 0x00; 988 action.pf_func = target; 989 action.op = req->op; 990 action.index = req->index; 991 action.match_id = req->match_id; 992 action.flow_key_alg = req->flow_key_alg; 993 994 if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule) 995 action = pfvf->def_ucast_rule->rx_action; 996 997 entry->action = *(u64 *)&action; 998 999 /* VTAG0 starts at 0th byte of LID_B. 1000 * VTAG1 starts at 4th byte of LID_B. 1001 */ 1002 entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | 1003 FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | 1004 FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | 1005 FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | 1006 FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) | 1007 FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) | 1008 FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) | 1009 FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4); 1010 } 1011 1012 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 1013 struct mcam_entry *entry, 1014 struct npc_install_flow_req *req, u16 target) 1015 { 1016 struct nix_tx_action action; 1017 u64 mask = ~0ULL; 1018 1019 /* If AF is installing then do not care about 1020 * PF_FUNC in Send Descriptor 1021 */ 1022 if (is_pffunc_af(req->hdr.pcifunc)) 1023 mask = 0; 1024 1025 npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), 1026 0, mask, 0, NIX_INTF_TX); 1027 1028 *(u64 *)&action = 0x00; 1029 action.op = req->op; 1030 action.index = req->index; 1031 action.match_id = req->match_id; 1032 1033 entry->action = *(u64 *)&action; 1034 1035 /* VTAG0 starts at 0th byte of LID_B. 1036 * VTAG1 starts at 4th byte of LID_B. 1037 */ 1038 entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | 1039 FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | 1040 FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | 1041 FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | 1042 FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) | 1043 FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) | 1044 FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) | 1045 FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24); 1046 } 1047 1048 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, 1049 int nixlf, struct rvu_pfvf *pfvf, 1050 struct npc_install_flow_req *req, 1051 struct npc_install_flow_rsp *rsp, bool enable, 1052 bool pf_set_vfs_mac) 1053 { 1054 struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; 1055 u64 features, installed_features, missing_features = 0; 1056 struct npc_mcam_write_entry_req write_req = { 0 }; 1057 struct npc_mcam *mcam = &rvu->hw->mcam; 1058 struct rvu_npc_mcam_rule dummy = { 0 }; 1059 struct rvu_npc_mcam_rule *rule; 1060 u16 owner = req->hdr.pcifunc; 1061 struct msg_rsp write_rsp; 1062 struct mcam_entry *entry; 1063 bool new = false; 1064 u16 entry_index; 1065 int err; 1066 1067 installed_features = req->features; 1068 features = req->features; 1069 entry = &write_req.entry_data; 1070 entry_index = req->entry; 1071 1072 npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, 1073 req->intf, blkaddr); 1074 1075 if (is_npc_intf_rx(req->intf)) 1076 npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); 1077 else 1078 npc_update_tx_entry(rvu, pfvf, entry, req, target); 1079 1080 /* Default unicast rules do not exist for TX */ 1081 if (is_npc_intf_tx(req->intf)) 1082 goto find_rule; 1083 1084 if (req->default_rule) { 1085 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf, 1086 NIXLF_UCAST_ENTRY); 1087 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index); 1088 } 1089 1090 /* update mcam entry with default unicast rule attributes */ 1091 if (def_ucast_rule && (req->default_rule && req->append)) { 1092 missing_features = (def_ucast_rule->features ^ features) & 1093 def_ucast_rule->features; 1094 if (missing_features) 1095 npc_update_flow(rvu, entry, missing_features, 1096 &def_ucast_rule->packet, 1097 &def_ucast_rule->mask, 1098 &dummy, req->intf, 1099 blkaddr); 1100 installed_features = req->features | missing_features; 1101 } 1102 1103 find_rule: 1104 rule = rvu_mcam_find_rule(mcam, entry_index); 1105 if (!rule) { 1106 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1107 if (!rule) 1108 return -ENOMEM; 1109 new = true; 1110 } 1111 1112 /* allocate new counter if rule has no counter */ 1113 if (!req->default_rule && req->set_cntr && !rule->has_cntr) 1114 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); 1115 1116 /* if user wants to delete an existing counter for a rule then 1117 * free the counter 1118 */ 1119 if (!req->set_cntr && rule->has_cntr) 1120 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1121 1122 write_req.hdr.pcifunc = owner; 1123 1124 /* AF owns the default rules so change the owner just to relax 1125 * the checks in rvu_mbox_handler_npc_mcam_write_entry 1126 */ 1127 if (req->default_rule) 1128 write_req.hdr.pcifunc = 0; 1129 1130 write_req.entry = entry_index; 1131 write_req.intf = req->intf; 1132 write_req.enable_entry = (u8)enable; 1133 /* if counter is available then clear and use it */ 1134 if (req->set_cntr && rule->has_cntr) { 1135 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00); 1136 write_req.set_cntr = 1; 1137 write_req.cntr = rule->cntr; 1138 } 1139 1140 /* update rule */ 1141 memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); 1142 memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); 1143 rule->entry = entry_index; 1144 memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); 1145 if (is_npc_intf_tx(req->intf)) 1146 memcpy(&rule->tx_action, &entry->action, 1147 sizeof(struct nix_tx_action)); 1148 rule->vtag_action = entry->vtag_action; 1149 rule->features = installed_features; 1150 rule->default_rule = req->default_rule; 1151 rule->owner = owner; 1152 rule->enable = enable; 1153 rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 1154 rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; 1155 rule->chan &= rule->chan_mask; 1156 if (is_npc_intf_tx(req->intf)) 1157 rule->intf = pfvf->nix_tx_intf; 1158 else 1159 rule->intf = pfvf->nix_rx_intf; 1160 1161 if (new) 1162 rvu_mcam_add_rule(mcam, rule); 1163 if (req->default_rule) 1164 pfvf->def_ucast_rule = rule; 1165 1166 /* write to mcam entry registers */ 1167 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, 1168 &write_rsp); 1169 if (err) { 1170 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1171 if (new) { 1172 list_del(&rule->list); 1173 kfree(rule); 1174 } 1175 return err; 1176 } 1177 1178 /* VF's MAC address is being changed via PF */ 1179 if (pf_set_vfs_mac) { 1180 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1181 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1182 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1183 } 1184 1185 if (test_bit(PF_SET_VF_CFG, &pfvf->flags) && 1186 req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) 1187 rule->vfvlan_cfg = true; 1188 1189 if (is_npc_intf_rx(req->intf) && req->match_id && 1190 (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS)) 1191 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc, 1192 req->index, req->match_id); 1193 1194 return 0; 1195 } 1196 1197 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, 1198 struct npc_install_flow_req *req, 1199 struct npc_install_flow_rsp *rsp) 1200 { 1201 bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK); 1202 struct rvu_switch *rswitch = &rvu->rswitch; 1203 int blkaddr, nixlf, err; 1204 struct rvu_pfvf *pfvf; 1205 bool pf_set_vfs_mac = false; 1206 bool enable = true; 1207 u16 target; 1208 1209 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1210 if (blkaddr < 0) { 1211 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1212 return NPC_MCAM_INVALID_REQ; 1213 } 1214 1215 if (!is_npc_interface_valid(rvu, req->intf)) 1216 return NPC_FLOW_INTF_INVALID; 1217 1218 if (from_vf && req->default_rule) 1219 return NPC_FLOW_VF_PERM_DENIED; 1220 1221 /* Each PF/VF info is maintained in struct rvu_pfvf. 1222 * rvu_pfvf for the target PF/VF needs to be retrieved 1223 * hence modify pcifunc accordingly. 1224 */ 1225 1226 /* AF installing for a PF/VF */ 1227 if (!req->hdr.pcifunc) 1228 target = req->vf; 1229 /* PF installing for its VF */ 1230 else if (!from_vf && req->vf) { 1231 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf; 1232 pf_set_vfs_mac = req->default_rule && 1233 (req->features & BIT_ULL(NPC_DMAC)); 1234 } 1235 /* msg received from PF/VF */ 1236 else 1237 target = req->hdr.pcifunc; 1238 1239 /* ignore chan_mask in case pf func is not AF, revisit later */ 1240 if (!is_pffunc_af(req->hdr.pcifunc)) 1241 req->chan_mask = 0xFFF; 1242 1243 err = npc_check_unsupported_flows(rvu, req->features, req->intf); 1244 if (err) 1245 return NPC_FLOW_NOT_SUPPORTED; 1246 1247 pfvf = rvu_get_pfvf(rvu, target); 1248 1249 /* PF installing for its VF */ 1250 if (req->hdr.pcifunc && !from_vf && req->vf) 1251 set_bit(PF_SET_VF_CFG, &pfvf->flags); 1252 1253 /* update req destination mac addr */ 1254 if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) && 1255 is_zero_ether_addr(req->packet.dmac)) { 1256 ether_addr_copy(req->packet.dmac, pfvf->mac_addr); 1257 eth_broadcast_addr((u8 *)&req->mask.dmac); 1258 } 1259 1260 /* Proceed if NIXLF is attached or not for TX rules */ 1261 err = nix_get_nixlf(rvu, target, &nixlf, NULL); 1262 if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) 1263 return NPC_FLOW_NO_NIXLF; 1264 1265 /* don't enable rule when nixlf not attached or initialized */ 1266 if (!(is_nixlf_attached(rvu, target) && 1267 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 1268 enable = false; 1269 1270 /* Packets reaching NPC in Tx path implies that a 1271 * NIXLF is properly setup and transmitting. 1272 * Hence rules can be enabled for Tx. 1273 */ 1274 if (is_npc_intf_tx(req->intf)) 1275 enable = true; 1276 1277 /* Do not allow requests from uninitialized VFs */ 1278 if (from_vf && !enable) 1279 return NPC_FLOW_VF_NOT_INIT; 1280 1281 /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ 1282 if (pf_set_vfs_mac && !enable) { 1283 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1284 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1285 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1286 return 0; 1287 } 1288 1289 mutex_lock(&rswitch->switch_lock); 1290 err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, 1291 req, rsp, enable, pf_set_vfs_mac); 1292 mutex_unlock(&rswitch->switch_lock); 1293 1294 return err; 1295 } 1296 1297 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule, 1298 u16 pcifunc) 1299 { 1300 struct npc_mcam_ena_dis_entry_req dis_req = { 0 }; 1301 struct msg_rsp dis_rsp; 1302 1303 if (rule->default_rule) 1304 return 0; 1305 1306 if (rule->has_cntr) 1307 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); 1308 1309 dis_req.hdr.pcifunc = pcifunc; 1310 dis_req.entry = rule->entry; 1311 1312 list_del(&rule->list); 1313 kfree(rule); 1314 1315 return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp); 1316 } 1317 1318 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, 1319 struct npc_delete_flow_req *req, 1320 struct msg_rsp *rsp) 1321 { 1322 struct npc_mcam *mcam = &rvu->hw->mcam; 1323 struct rvu_npc_mcam_rule *iter, *tmp; 1324 u16 pcifunc = req->hdr.pcifunc; 1325 struct list_head del_list; 1326 1327 INIT_LIST_HEAD(&del_list); 1328 1329 mutex_lock(&mcam->lock); 1330 list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) { 1331 if (iter->owner == pcifunc) { 1332 /* All rules */ 1333 if (req->all) { 1334 list_move_tail(&iter->list, &del_list); 1335 /* Range of rules */ 1336 } else if (req->end && iter->entry >= req->start && 1337 iter->entry <= req->end) { 1338 list_move_tail(&iter->list, &del_list); 1339 /* single rule */ 1340 } else if (req->entry == iter->entry) { 1341 list_move_tail(&iter->list, &del_list); 1342 break; 1343 } 1344 } 1345 } 1346 mutex_unlock(&mcam->lock); 1347 1348 list_for_each_entry_safe(iter, tmp, &del_list, list) { 1349 u16 entry = iter->entry; 1350 1351 /* clear the mcam entry target pcifunc */ 1352 mcam->entry2target_pffunc[entry] = 0x0; 1353 if (npc_delete_flow(rvu, iter, pcifunc)) 1354 dev_err(rvu->dev, "rule deletion failed for entry:%u", 1355 entry); 1356 } 1357 1358 return 0; 1359 } 1360 1361 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, 1362 struct rvu_npc_mcam_rule *rule, 1363 struct rvu_pfvf *pfvf) 1364 { 1365 struct npc_mcam_write_entry_req write_req = { 0 }; 1366 struct mcam_entry *entry = &write_req.entry_data; 1367 struct npc_mcam *mcam = &rvu->hw->mcam; 1368 struct msg_rsp rsp; 1369 u8 intf, enable; 1370 int err; 1371 1372 ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); 1373 1374 npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, 1375 entry, &intf, &enable); 1376 1377 npc_update_entry(rvu, NPC_DMAC, entry, 1378 ether_addr_to_u64(pfvf->mac_addr), 0, 1379 0xffffffffffffull, 0, intf); 1380 1381 write_req.hdr.pcifunc = rule->owner; 1382 write_req.entry = rule->entry; 1383 write_req.intf = pfvf->nix_rx_intf; 1384 1385 mutex_unlock(&mcam->lock); 1386 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); 1387 mutex_lock(&mcam->lock); 1388 1389 return err; 1390 } 1391 1392 void npc_mcam_enable_flows(struct rvu *rvu, u16 target) 1393 { 1394 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target); 1395 struct rvu_npc_mcam_rule *def_ucast_rule; 1396 struct npc_mcam *mcam = &rvu->hw->mcam; 1397 struct rvu_npc_mcam_rule *rule; 1398 int blkaddr, bank, index; 1399 u64 def_action; 1400 1401 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1402 if (blkaddr < 0) 1403 return; 1404 1405 def_ucast_rule = pfvf->def_ucast_rule; 1406 1407 mutex_lock(&mcam->lock); 1408 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1409 if (is_npc_intf_rx(rule->intf) && 1410 rule->rx_action.pf_func == target && !rule->enable) { 1411 if (rule->default_rule) { 1412 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1413 rule->entry, true); 1414 rule->enable = true; 1415 continue; 1416 } 1417 1418 if (rule->vfvlan_cfg) 1419 npc_update_dmac_value(rvu, blkaddr, rule, pfvf); 1420 1421 if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) { 1422 if (!def_ucast_rule) 1423 continue; 1424 /* Use default unicast entry action */ 1425 rule->rx_action = def_ucast_rule->rx_action; 1426 def_action = *(u64 *)&def_ucast_rule->rx_action; 1427 bank = npc_get_bank(mcam, rule->entry); 1428 rvu_write64(rvu, blkaddr, 1429 NPC_AF_MCAMEX_BANKX_ACTION 1430 (rule->entry, bank), def_action); 1431 } 1432 1433 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1434 rule->entry, true); 1435 rule->enable = true; 1436 } 1437 } 1438 1439 /* Enable MCAM entries installed by PF with target as VF pcifunc */ 1440 for (index = 0; index < mcam->bmap_entries; index++) { 1441 if (mcam->entry2target_pffunc[index] == target) 1442 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1443 index, true); 1444 } 1445 mutex_unlock(&mcam->lock); 1446 } 1447 1448 void npc_mcam_disable_flows(struct rvu *rvu, u16 target) 1449 { 1450 struct npc_mcam *mcam = &rvu->hw->mcam; 1451 int blkaddr, index; 1452 1453 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1454 if (blkaddr < 0) 1455 return; 1456 1457 mutex_lock(&mcam->lock); 1458 /* Disable MCAM entries installed by PF with target as VF pcifunc */ 1459 for (index = 0; index < mcam->bmap_entries; index++) { 1460 if (mcam->entry2target_pffunc[index] == target) 1461 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1462 index, false); 1463 } 1464 mutex_unlock(&mcam->lock); 1465 } 1466 1467 /* single drop on non hit rule starting from 0th index. This an extension 1468 * to RPM mac filter to support more rules. 1469 */ 1470 int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, 1471 u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, 1472 u64 bcast_mcast_val, u64 bcast_mcast_mask) 1473 { 1474 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 1475 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 1476 struct npc_mcam_write_entry_req req = { 0 }; 1477 struct npc_mcam *mcam = &rvu->hw->mcam; 1478 struct rvu_npc_mcam_rule *rule; 1479 struct msg_rsp rsp; 1480 bool enabled; 1481 int blkaddr; 1482 int err; 1483 1484 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1485 if (blkaddr < 0) { 1486 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1487 return -ENODEV; 1488 } 1489 1490 /* Bail out if no exact match support */ 1491 if (!rvu_npc_exact_has_match_table(rvu)) { 1492 dev_info(rvu->dev, "%s: No support for exact match feature\n", __func__); 1493 return -EINVAL; 1494 } 1495 1496 /* If 0th entry is already used, return err */ 1497 enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_idx); 1498 if (enabled) { 1499 dev_err(rvu->dev, "%s: failed to add single drop on non hit rule at %d th index\n", 1500 __func__, mcam_idx); 1501 return -EINVAL; 1502 } 1503 1504 /* Add this entry to mcam rules list */ 1505 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1506 if (!rule) 1507 return -ENOMEM; 1508 1509 /* Disable rule by default. Enable rule when first dmac filter is 1510 * installed 1511 */ 1512 rule->enable = false; 1513 rule->chan = chan_val; 1514 rule->chan_mask = chan_mask; 1515 rule->entry = mcam_idx; 1516 rvu_mcam_add_rule(mcam, rule); 1517 1518 /* Reserve slot 0 */ 1519 npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx); 1520 1521 /* Allocate counter for this single drop on non hit rule */ 1522 cntr_req.hdr.pcifunc = 0; /* AF request */ 1523 cntr_req.contig = true; 1524 cntr_req.count = 1; 1525 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 1526 if (err) { 1527 dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n", 1528 __func__, err); 1529 return -EFAULT; 1530 } 1531 *counter_idx = cntr_rsp.cntr; 1532 1533 /* Fill in fields for this mcam entry */ 1534 npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0, 1535 exact_mask, 0, NIX_INTF_RX); 1536 npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0, 1537 chan_mask, 0, NIX_INTF_RX); 1538 npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0, 1539 bcast_mcast_mask, 0, NIX_INTF_RX); 1540 1541 req.intf = NIX_INTF_RX; 1542 req.set_cntr = true; 1543 req.cntr = cntr_rsp.cntr; 1544 req.entry = mcam_idx; 1545 1546 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp); 1547 if (err) { 1548 dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n", 1549 __func__, mcam_idx); 1550 return err; 1551 } 1552 1553 dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", 1554 __func__, mcam_idx, req.cntr); 1555 1556 /* disable entry at Bank 0, index 0 */ 1557 npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false); 1558 1559 return 0; 1560 } 1561