1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ 3 4 #include <linux/kernel.h> 5 #include <linux/io.h> 6 #include <linux/iopoll.h> 7 #include <linux/etherdevice.h> 8 #include <linux/platform_device.h> 9 #include <linux/if_ether.h> 10 #include <linux/if_vlan.h> 11 #include <net/dst_metadata.h> 12 #include <net/dsa.h> 13 #include "mtk_eth_soc.h" 14 #include "mtk_ppe.h" 15 #include "mtk_ppe_regs.h" 16 17 static DEFINE_SPINLOCK(ppe_lock); 18 19 static const struct rhashtable_params mtk_flow_l2_ht_params = { 20 .head_offset = offsetof(struct mtk_flow_entry, l2_node), 21 .key_offset = offsetof(struct mtk_flow_entry, data.bridge), 22 .key_len = offsetof(struct mtk_foe_bridge, key_end), 23 .automatic_shrinking = true, 24 }; 25 26 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) 27 { 28 writel(val, ppe->base + reg); 29 } 30 31 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) 32 { 33 return readl(ppe->base + reg); 34 } 35 36 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) 37 { 38 u32 val; 39 40 val = ppe_r32(ppe, reg); 41 val &= ~mask; 42 val |= set; 43 ppe_w32(ppe, reg, val); 44 45 return val; 46 } 47 48 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) 49 { 50 return ppe_m32(ppe, reg, 0, val); 51 } 52 53 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) 54 { 55 return ppe_m32(ppe, reg, val, 0); 56 } 57 58 static u32 mtk_eth_timestamp(struct mtk_eth *eth) 59 { 60 return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); 61 } 62 63 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) 64 { 65 int ret; 66 u32 val; 67 68 ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, 69 !(val & MTK_PPE_GLO_CFG_BUSY), 70 20, MTK_PPE_WAIT_TIMEOUT_US); 71 72 if (ret) 73 dev_err(ppe->dev, "PPE table busy"); 74 75 return ret; 76 } 77 78 static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe) 79 { 80 int ret; 81 u32 val; 82 83 ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val, 84 !(val & MTK_PPE_MIB_SER_CR_ST), 85 20, MTK_PPE_WAIT_TIMEOUT_US); 86 87 if (ret) 88 dev_err(ppe->dev, "MIB table busy"); 89 90 return ret; 91 } 92 93 static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets) 94 { 95 u32 byte_cnt_low, byte_cnt_high, pkt_cnt_low, pkt_cnt_high; 96 u32 val, cnt_r0, cnt_r1, cnt_r2; 97 int ret; 98 99 val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST; 100 ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val); 101 102 ret = mtk_ppe_mib_wait_busy(ppe); 103 if (ret) 104 return ret; 105 106 cnt_r0 = readl(ppe->base + MTK_PPE_MIB_SER_R0); 107 cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1); 108 cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2); 109 110 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0); 111 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1); 112 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1); 113 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2); 114 *bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low; 115 *packets = (pkt_cnt_high << 16) | pkt_cnt_low; 116 117 return 0; 118 } 119 120 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) 121 { 122 ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 123 ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 124 } 125 126 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) 127 { 128 mtk_ppe_cache_clear(ppe); 129 130 ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, 131 enable * MTK_PPE_CACHE_CTL_EN); 132 } 133 134 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) 135 { 136 u32 hv1, hv2, hv3; 137 u32 hash; 138 139 switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { 140 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 141 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 142 hv1 = e->ipv4.orig.ports; 143 hv2 = e->ipv4.orig.dest_ip; 144 hv3 = e->ipv4.orig.src_ip; 145 break; 146 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 147 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 148 hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; 149 hv1 ^= e->ipv6.ports; 150 151 hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; 152 hv2 ^= e->ipv6.dest_ip[0]; 153 154 hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; 155 hv3 ^= e->ipv6.src_ip[0]; 156 break; 157 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 158 case MTK_PPE_PKT_TYPE_IPV6_6RD: 159 default: 160 WARN_ON_ONCE(1); 161 return MTK_PPE_HASH_MASK; 162 } 163 164 hash = (hv1 & hv2) | ((~hv1) & hv3); 165 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 166 hash ^= hv1 ^ hv2 ^ hv3; 167 hash ^= hash >> 16; 168 hash <<= (ffs(eth->soc->hash_offset) - 1); 169 hash &= MTK_PPE_ENTRIES - 1; 170 171 return hash; 172 } 173 174 static inline struct mtk_foe_mac_info * 175 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 176 { 177 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 178 179 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 180 return &entry->bridge.l2; 181 182 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 183 return &entry->ipv6.l2; 184 185 return &entry->ipv4.l2; 186 } 187 188 static inline u32 * 189 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 190 { 191 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 192 193 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 194 return &entry->bridge.ib2; 195 196 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 197 return &entry->ipv6.ib2; 198 199 return &entry->ipv4.ib2; 200 } 201 202 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, 203 int type, int l4proto, u8 pse_port, u8 *src_mac, 204 u8 *dest_mac) 205 { 206 struct mtk_foe_mac_info *l2; 207 u32 ports_pad, val; 208 209 memset(entry, 0, sizeof(*entry)); 210 211 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 212 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 213 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | 214 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 215 MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; 216 entry->ib1 = val; 217 218 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | 219 FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); 220 } else { 221 int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f; 222 223 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 224 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | 225 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 226 MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; 227 entry->ib1 = val; 228 229 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | 230 FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) | 231 FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); 232 } 233 234 if (is_multicast_ether_addr(dest_mac)) 235 val |= mtk_get_ib2_multicast_mask(eth); 236 237 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 238 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 239 entry->ipv4.orig.ports = ports_pad; 240 if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 241 entry->ipv6.ports = ports_pad; 242 243 if (type == MTK_PPE_PKT_TYPE_BRIDGE) { 244 ether_addr_copy(entry->bridge.src_mac, src_mac); 245 ether_addr_copy(entry->bridge.dest_mac, dest_mac); 246 entry->bridge.ib2 = val; 247 l2 = &entry->bridge.l2; 248 } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { 249 entry->ipv6.ib2 = val; 250 l2 = &entry->ipv6.l2; 251 } else { 252 entry->ipv4.ib2 = val; 253 l2 = &entry->ipv4.l2; 254 } 255 256 l2->dest_mac_hi = get_unaligned_be32(dest_mac); 257 l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4); 258 l2->src_mac_hi = get_unaligned_be32(src_mac); 259 l2->src_mac_lo = get_unaligned_be16(src_mac + 4); 260 261 if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 262 l2->etype = ETH_P_IPV6; 263 else 264 l2->etype = ETH_P_IP; 265 266 return 0; 267 } 268 269 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, 270 struct mtk_foe_entry *entry, u8 port) 271 { 272 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 273 u32 val = *ib2; 274 275 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 276 val &= ~MTK_FOE_IB2_DEST_PORT_V2; 277 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); 278 } else { 279 val &= ~MTK_FOE_IB2_DEST_PORT; 280 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); 281 } 282 *ib2 = val; 283 284 return 0; 285 } 286 287 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, 288 struct mtk_foe_entry *entry, bool egress, 289 __be32 src_addr, __be16 src_port, 290 __be32 dest_addr, __be16 dest_port) 291 { 292 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 293 struct mtk_ipv4_tuple *t; 294 295 switch (type) { 296 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 297 if (egress) { 298 t = &entry->ipv4.new; 299 break; 300 } 301 fallthrough; 302 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 303 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 304 t = &entry->ipv4.orig; 305 break; 306 case MTK_PPE_PKT_TYPE_IPV6_6RD: 307 entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); 308 entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); 309 return 0; 310 default: 311 WARN_ON_ONCE(1); 312 return -EINVAL; 313 } 314 315 t->src_ip = be32_to_cpu(src_addr); 316 t->dest_ip = be32_to_cpu(dest_addr); 317 318 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 319 return 0; 320 321 t->src_port = be16_to_cpu(src_port); 322 t->dest_port = be16_to_cpu(dest_port); 323 324 return 0; 325 } 326 327 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, 328 struct mtk_foe_entry *entry, 329 __be32 *src_addr, __be16 src_port, 330 __be32 *dest_addr, __be16 dest_port) 331 { 332 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 333 u32 *src, *dest; 334 int i; 335 336 switch (type) { 337 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 338 src = entry->dslite.tunnel_src_ip; 339 dest = entry->dslite.tunnel_dest_ip; 340 break; 341 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 342 case MTK_PPE_PKT_TYPE_IPV6_6RD: 343 entry->ipv6.src_port = be16_to_cpu(src_port); 344 entry->ipv6.dest_port = be16_to_cpu(dest_port); 345 fallthrough; 346 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 347 src = entry->ipv6.src_ip; 348 dest = entry->ipv6.dest_ip; 349 break; 350 default: 351 WARN_ON_ONCE(1); 352 return -EINVAL; 353 } 354 355 for (i = 0; i < 4; i++) 356 src[i] = be32_to_cpu(src_addr[i]); 357 for (i = 0; i < 4; i++) 358 dest[i] = be32_to_cpu(dest_addr[i]); 359 360 return 0; 361 } 362 363 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, 364 int port) 365 { 366 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 367 368 l2->etype = BIT(port); 369 370 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) 371 entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); 372 else 373 l2->etype |= BIT(8); 374 375 entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); 376 377 return 0; 378 } 379 380 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, 381 int vid) 382 { 383 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 384 385 switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { 386 case 0: 387 entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | 388 mtk_prep_ib1_vlan_layer(eth, 1); 389 l2->vlan1 = vid; 390 return 0; 391 case 1: 392 if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { 393 l2->vlan1 = vid; 394 l2->etype |= BIT(8); 395 } else { 396 l2->vlan2 = vid; 397 entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); 398 } 399 return 0; 400 default: 401 return -ENOSPC; 402 } 403 } 404 405 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, 406 int sid) 407 { 408 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 409 410 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || 411 (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) 412 l2->etype = ETH_P_PPP_SES; 413 414 entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); 415 l2->pppoe_id = sid; 416 417 return 0; 418 } 419 420 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, 421 int wdma_idx, int txq, int bss, int wcid) 422 { 423 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 424 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 425 426 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 427 *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; 428 *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | 429 MTK_FOE_IB2_WDMA_WINFO_V2; 430 l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | 431 FIELD_PREP(MTK_FOE_WINFO_BSS, bss); 432 } else { 433 *ib2 &= ~MTK_FOE_IB2_PORT_MG; 434 *ib2 |= MTK_FOE_IB2_WDMA_WINFO; 435 if (wdma_idx) 436 *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; 437 l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | 438 FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | 439 FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); 440 } 441 442 return 0; 443 } 444 445 int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry, 446 unsigned int queue) 447 { 448 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 449 450 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 451 *ib2 &= ~MTK_FOE_IB2_QID_V2; 452 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue); 453 *ib2 |= MTK_FOE_IB2_PSE_QOS_V2; 454 } else { 455 *ib2 &= ~MTK_FOE_IB2_QID; 456 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue); 457 *ib2 |= MTK_FOE_IB2_PSE_QOS; 458 } 459 460 return 0; 461 } 462 463 static bool 464 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, 465 struct mtk_foe_entry *data) 466 { 467 int type, len; 468 469 if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) 470 return false; 471 472 type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); 473 if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) 474 len = offsetof(struct mtk_foe_entry, ipv6._rsv); 475 else 476 len = offsetof(struct mtk_foe_entry, ipv4.ib2); 477 478 return !memcmp(&entry->data.data, &data->data, len - 4); 479 } 480 481 static void 482 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 483 { 484 struct hlist_head *head; 485 struct hlist_node *tmp; 486 487 if (entry->type == MTK_FLOW_TYPE_L2) { 488 rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, 489 mtk_flow_l2_ht_params); 490 491 head = &entry->l2_flows; 492 hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) 493 __mtk_foe_entry_clear(ppe, entry); 494 return; 495 } 496 497 hlist_del_init(&entry->list); 498 if (entry->hash != 0xffff) { 499 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); 500 501 hwe->ib1 &= ~MTK_FOE_IB1_STATE; 502 hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); 503 dma_wmb(); 504 mtk_ppe_cache_clear(ppe); 505 506 if (ppe->accounting) { 507 struct mtk_foe_accounting *acct; 508 509 acct = ppe->acct_table + entry->hash * sizeof(*acct); 510 acct->packets = 0; 511 acct->bytes = 0; 512 } 513 } 514 entry->hash = 0xffff; 515 516 if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) 517 return; 518 519 hlist_del_init(&entry->l2_data.list); 520 kfree(entry); 521 } 522 523 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) 524 { 525 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 526 u16 now = mtk_eth_timestamp(ppe->eth); 527 u16 timestamp = ib1 & ib1_ts_mask; 528 529 if (timestamp > now) 530 return ib1_ts_mask + 1 - timestamp + now; 531 else 532 return now - timestamp; 533 } 534 535 static void 536 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 537 { 538 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 539 struct mtk_flow_entry *cur; 540 struct mtk_foe_entry *hwe; 541 struct hlist_node *tmp; 542 int idle; 543 544 idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 545 hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { 546 int cur_idle; 547 u32 ib1; 548 549 hwe = mtk_foe_get_entry(ppe, cur->hash); 550 ib1 = READ_ONCE(hwe->ib1); 551 552 if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { 553 cur->hash = 0xffff; 554 __mtk_foe_entry_clear(ppe, cur); 555 continue; 556 } 557 558 cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); 559 if (cur_idle >= idle) 560 continue; 561 562 idle = cur_idle; 563 entry->data.ib1 &= ~ib1_ts_mask; 564 entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; 565 } 566 } 567 568 static void 569 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 570 { 571 struct mtk_foe_entry foe = {}; 572 struct mtk_foe_entry *hwe; 573 574 spin_lock_bh(&ppe_lock); 575 576 if (entry->type == MTK_FLOW_TYPE_L2) { 577 mtk_flow_entry_update_l2(ppe, entry); 578 goto out; 579 } 580 581 if (entry->hash == 0xffff) 582 goto out; 583 584 hwe = mtk_foe_get_entry(ppe, entry->hash); 585 memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); 586 if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { 587 entry->hash = 0xffff; 588 goto out; 589 } 590 591 entry->data.ib1 = foe.ib1; 592 593 out: 594 spin_unlock_bh(&ppe_lock); 595 } 596 597 static void 598 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, 599 u16 hash) 600 { 601 struct mtk_eth *eth = ppe->eth; 602 u16 timestamp = mtk_eth_timestamp(eth); 603 struct mtk_foe_entry *hwe; 604 u32 val; 605 606 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 607 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; 608 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, 609 timestamp); 610 } else { 611 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; 612 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, 613 timestamp); 614 } 615 616 hwe = mtk_foe_get_entry(ppe, hash); 617 memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); 618 wmb(); 619 hwe->ib1 = entry->ib1; 620 621 if (ppe->accounting) { 622 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) 623 val = MTK_FOE_IB2_MIB_CNT_V2; 624 else 625 val = MTK_FOE_IB2_MIB_CNT; 626 *mtk_foe_entry_ib2(eth, hwe) |= val; 627 } 628 629 dma_wmb(); 630 631 mtk_ppe_cache_clear(ppe); 632 } 633 634 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 635 { 636 spin_lock_bh(&ppe_lock); 637 __mtk_foe_entry_clear(ppe, entry); 638 spin_unlock_bh(&ppe_lock); 639 } 640 641 static int 642 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 643 { 644 struct mtk_flow_entry *prev; 645 646 entry->type = MTK_FLOW_TYPE_L2; 647 648 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &entry->l2_node, 649 mtk_flow_l2_ht_params); 650 if (likely(!prev)) 651 return 0; 652 653 if (IS_ERR(prev)) 654 return PTR_ERR(prev); 655 656 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 657 &entry->l2_node, mtk_flow_l2_ht_params); 658 } 659 660 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 661 { 662 const struct mtk_soc_data *soc = ppe->eth->soc; 663 int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); 664 u32 hash; 665 666 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 667 return mtk_foe_entry_commit_l2(ppe, entry); 668 669 hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); 670 entry->hash = 0xffff; 671 spin_lock_bh(&ppe_lock); 672 hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); 673 spin_unlock_bh(&ppe_lock); 674 675 return 0; 676 } 677 678 static void 679 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, 680 u16 hash) 681 { 682 const struct mtk_soc_data *soc = ppe->eth->soc; 683 struct mtk_flow_entry *flow_info; 684 struct mtk_foe_entry foe = {}, *hwe; 685 struct mtk_foe_mac_info *l2; 686 u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; 687 int type; 688 689 flow_info = kzalloc(sizeof(*flow_info), GFP_ATOMIC); 690 if (!flow_info) 691 return; 692 693 flow_info->l2_data.base_flow = entry; 694 flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; 695 flow_info->hash = hash; 696 hlist_add_head(&flow_info->list, 697 &ppe->foe_flow[hash / soc->hash_offset]); 698 hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); 699 700 hwe = mtk_foe_get_entry(ppe, hash); 701 memcpy(&foe, hwe, soc->foe_entry_size); 702 foe.ib1 &= ib1_mask; 703 foe.ib1 |= entry->data.ib1 & ~ib1_mask; 704 705 l2 = mtk_foe_entry_l2(ppe->eth, &foe); 706 memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); 707 708 type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); 709 if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) 710 memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); 711 else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) 712 l2->etype = ETH_P_IPV6; 713 714 *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; 715 716 __mtk_foe_entry_commit(ppe, &foe, hash); 717 } 718 719 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) 720 { 721 const struct mtk_soc_data *soc = ppe->eth->soc; 722 struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; 723 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); 724 struct mtk_flow_entry *entry; 725 struct mtk_foe_bridge key = {}; 726 struct hlist_node *n; 727 struct ethhdr *eh; 728 bool found = false; 729 u8 *tag; 730 731 spin_lock_bh(&ppe_lock); 732 733 if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) 734 goto out; 735 736 hlist_for_each_entry_safe(entry, n, head, list) { 737 if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { 738 if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == 739 MTK_FOE_STATE_BIND)) 740 continue; 741 742 entry->hash = 0xffff; 743 __mtk_foe_entry_clear(ppe, entry); 744 continue; 745 } 746 747 if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { 748 if (entry->hash != 0xffff) 749 entry->hash = 0xffff; 750 continue; 751 } 752 753 entry->hash = hash; 754 __mtk_foe_entry_commit(ppe, &entry->data, hash); 755 found = true; 756 } 757 758 if (found) 759 goto out; 760 761 eh = eth_hdr(skb); 762 ether_addr_copy(key.dest_mac, eh->h_dest); 763 ether_addr_copy(key.src_mac, eh->h_source); 764 tag = skb->data - 2; 765 key.vlan = 0; 766 switch (skb->protocol) { 767 #if IS_ENABLED(CONFIG_NET_DSA) 768 case htons(ETH_P_XDSA): 769 if (!netdev_uses_dsa(skb->dev) || 770 skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 771 goto out; 772 773 if (!skb_metadata_dst(skb)) 774 tag += 4; 775 776 if (get_unaligned_be16(tag) != ETH_P_8021Q) 777 break; 778 779 fallthrough; 780 #endif 781 case htons(ETH_P_8021Q): 782 key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; 783 break; 784 default: 785 break; 786 } 787 788 entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); 789 if (!entry) 790 goto out; 791 792 mtk_foe_entry_commit_subflow(ppe, entry, hash); 793 794 out: 795 spin_unlock_bh(&ppe_lock); 796 } 797 798 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 799 { 800 mtk_flow_entry_update(ppe, entry); 801 802 return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 803 } 804 805 int mtk_ppe_prepare_reset(struct mtk_ppe *ppe) 806 { 807 if (!ppe) 808 return -EINVAL; 809 810 /* disable KA */ 811 ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); 812 ppe_clear(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); 813 ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0); 814 usleep_range(10000, 11000); 815 816 /* set KA timer to maximum */ 817 ppe_set(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); 818 ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0xffffffff); 819 820 /* set KA tick select */ 821 ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_TICK_SEL); 822 ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); 823 usleep_range(10000, 11000); 824 825 /* disable scan mode */ 826 ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_SCAN_MODE); 827 usleep_range(10000, 11000); 828 829 return mtk_ppe_wait_busy(ppe); 830 } 831 832 struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index, 833 struct mtk_foe_accounting *diff) 834 { 835 struct mtk_foe_accounting *acct; 836 int size = sizeof(struct mtk_foe_accounting); 837 u64 bytes, packets; 838 839 if (!ppe->accounting) 840 return NULL; 841 842 if (mtk_mib_entry_read(ppe, index, &bytes, &packets)) 843 return NULL; 844 845 acct = ppe->acct_table + index * size; 846 847 acct->bytes += bytes; 848 acct->packets += packets; 849 850 if (diff) { 851 diff->bytes = bytes; 852 diff->packets = packets; 853 } 854 855 return acct; 856 } 857 858 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index) 859 { 860 bool accounting = eth->soc->has_accounting; 861 const struct mtk_soc_data *soc = eth->soc; 862 struct mtk_foe_accounting *acct; 863 struct device *dev = eth->dev; 864 struct mtk_mib_entry *mib; 865 struct mtk_ppe *ppe; 866 u32 foe_flow_size; 867 void *foe; 868 869 ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); 870 if (!ppe) 871 return NULL; 872 873 rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); 874 875 /* need to allocate a separate device, since it PPE DMA access is 876 * not coherent. 877 */ 878 ppe->base = base; 879 ppe->eth = eth; 880 ppe->dev = dev; 881 ppe->version = eth->soc->offload_version; 882 ppe->accounting = accounting; 883 884 foe = dmam_alloc_coherent(ppe->dev, 885 MTK_PPE_ENTRIES * soc->foe_entry_size, 886 &ppe->foe_phys, GFP_KERNEL); 887 if (!foe) 888 goto err_free_l2_flows; 889 890 ppe->foe_table = foe; 891 892 foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * 893 sizeof(*ppe->foe_flow); 894 ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); 895 if (!ppe->foe_flow) 896 goto err_free_l2_flows; 897 898 if (accounting) { 899 mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib), 900 &ppe->mib_phys, GFP_KERNEL); 901 if (!mib) 902 return NULL; 903 904 ppe->mib_table = mib; 905 906 acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct), 907 GFP_KERNEL); 908 909 if (!acct) 910 return NULL; 911 912 ppe->acct_table = acct; 913 } 914 915 mtk_ppe_debugfs_init(ppe, index); 916 917 return ppe; 918 919 err_free_l2_flows: 920 rhashtable_destroy(&ppe->l2_flows); 921 return NULL; 922 } 923 924 void mtk_ppe_deinit(struct mtk_eth *eth) 925 { 926 int i; 927 928 for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) { 929 if (!eth->ppe[i]) 930 return; 931 rhashtable_destroy(ð->ppe[i]->l2_flows); 932 } 933 } 934 935 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) 936 { 937 static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; 938 int i, k; 939 940 memset(ppe->foe_table, 0, 941 MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); 942 943 if (!IS_ENABLED(CONFIG_SOC_MT7621)) 944 return; 945 946 /* skip all entries that cross the 1024 byte boundary */ 947 for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { 948 for (k = 0; k < ARRAY_SIZE(skip); k++) { 949 struct mtk_foe_entry *hwe; 950 951 hwe = mtk_foe_get_entry(ppe, i + skip[k]); 952 hwe->ib1 |= MTK_FOE_IB1_STATIC; 953 } 954 } 955 } 956 957 void mtk_ppe_start(struct mtk_ppe *ppe) 958 { 959 u32 val; 960 961 if (!ppe) 962 return; 963 964 mtk_ppe_init_foe_table(ppe); 965 ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); 966 967 val = MTK_PPE_TB_CFG_ENTRY_80B | 968 MTK_PPE_TB_CFG_AGE_NON_L4 | 969 MTK_PPE_TB_CFG_AGE_UNBIND | 970 MTK_PPE_TB_CFG_AGE_TCP | 971 MTK_PPE_TB_CFG_AGE_UDP | 972 MTK_PPE_TB_CFG_AGE_TCP_FIN | 973 FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, 974 MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | 975 FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, 976 MTK_PPE_KEEPALIVE_DISABLE) | 977 FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | 978 FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, 979 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | 980 FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, 981 MTK_PPE_ENTRIES_SHIFT); 982 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 983 val |= MTK_PPE_TB_CFG_INFO_SEL; 984 ppe_w32(ppe, MTK_PPE_TB_CFG, val); 985 986 ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, 987 MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); 988 989 mtk_ppe_cache_enable(ppe, true); 990 991 val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | 992 MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | 993 MTK_PPE_FLOW_CFG_IP6_6RD | 994 MTK_PPE_FLOW_CFG_IP4_NAT | 995 MTK_PPE_FLOW_CFG_IP4_NAPT | 996 MTK_PPE_FLOW_CFG_IP4_DSLITE | 997 MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; 998 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 999 val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | 1000 MTK_PPE_MD_TOAP_BYP_CRSN1 | 1001 MTK_PPE_MD_TOAP_BYP_CRSN2 | 1002 MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; 1003 else 1004 val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | 1005 MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; 1006 ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); 1007 1008 val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | 1009 FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); 1010 ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); 1011 1012 val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | 1013 FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); 1014 ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); 1015 1016 val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 1017 FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); 1018 ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); 1019 1020 val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; 1021 ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); 1022 1023 val = MTK_PPE_BIND_LIMIT1_FULL | 1024 FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); 1025 ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); 1026 1027 val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | 1028 FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); 1029 ppe_w32(ppe, MTK_PPE_BIND_RATE, val); 1030 1031 /* enable PPE */ 1032 val = MTK_PPE_GLO_CFG_EN | 1033 MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | 1034 MTK_PPE_GLO_CFG_IP4_CS_DROP | 1035 MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; 1036 ppe_w32(ppe, MTK_PPE_GLO_CFG, val); 1037 1038 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); 1039 1040 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { 1041 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); 1042 ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); 1043 } 1044 1045 if (ppe->accounting && ppe->mib_phys) { 1046 ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, ppe->mib_phys); 1047 ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN, 1048 MTK_PPE_MIB_CFG_EN); 1049 ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR, 1050 MTK_PPE_MIB_CFG_RD_CLR); 1051 ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN, 1052 MTK_PPE_MIB_CFG_RD_CLR); 1053 } 1054 } 1055 1056 int mtk_ppe_stop(struct mtk_ppe *ppe) 1057 { 1058 u32 val; 1059 int i; 1060 1061 if (!ppe) 1062 return 0; 1063 1064 for (i = 0; i < MTK_PPE_ENTRIES; i++) { 1065 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); 1066 1067 hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, 1068 MTK_FOE_STATE_INVALID); 1069 } 1070 1071 mtk_ppe_cache_enable(ppe, false); 1072 1073 /* disable offload engine */ 1074 ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); 1075 ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0); 1076 1077 /* disable aging */ 1078 val = MTK_PPE_TB_CFG_AGE_NON_L4 | 1079 MTK_PPE_TB_CFG_AGE_UNBIND | 1080 MTK_PPE_TB_CFG_AGE_TCP | 1081 MTK_PPE_TB_CFG_AGE_UDP | 1082 MTK_PPE_TB_CFG_AGE_TCP_FIN; 1083 ppe_clear(ppe, MTK_PPE_TB_CFG, val); 1084 1085 return mtk_ppe_wait_busy(ppe); 1086 } 1087