1 /* $OpenBSD: if_ogx.c,v 1.1 2019/11/04 14:58:40 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2019 Visa Hankala 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Driver for OCTEON III network processor. 21 */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/atomic.h> 26 #include <sys/device.h> 27 #include <sys/ioctl.h> 28 #include <sys/rwlock.h> 29 #include <sys/socket.h> 30 #include <sys/stdint.h> 31 32 #include <net/if.h> 33 #include <net/if_media.h> 34 #include <netinet/in.h> 35 #include <netinet/if_ether.h> 36 37 #include <dev/mii/mii.h> 38 #include <dev/mii/miivar.h> 39 40 #include <dev/ofw/fdt.h> 41 #include <dev/ofw/openfirm.h> 42 43 #include <machine/bus.h> 44 #include <machine/fdt.h> 45 #include <machine/octeonvar.h> 46 #include <machine/octeon_model.h> 47 48 #include <octeon/dev/cn30xxsmivar.h> 49 #include <octeon/dev/ogxreg.h> 50 #include <octeon/dev/ogxvar.h> 51 52 struct ogx_link_ops; 53 54 struct ogx_softc { 55 struct device sc_dev; 56 struct arpcom sc_ac; 57 unsigned int sc_bgxid; 58 unsigned int sc_lmacid; 59 unsigned int sc_ipdport; 60 unsigned int sc_pkomac; 61 unsigned int sc_rxused; 62 unsigned int sc_txfree; 63 64 struct ogx_node *sc_node; 65 unsigned int sc_unit; /* logical unit within node */ 66 67 struct mii_data sc_mii; 68 #define sc_media sc_mii.mii_media 69 struct timeout sc_tick; 70 struct cn30xxsmi_softc *sc_smi; 71 72 struct timeout sc_rxrefill; 73 void *sc_rx_ih; 74 void *sc_tx_ih; 75 76 bus_space_tag_t sc_iot; 77 bus_space_handle_t sc_port_ioh; 78 bus_space_handle_t sc_nexus_ioh; 79 80 struct fpa3aura sc_pkt_aura; 81 const struct ogx_link_ops *sc_link_ops; 82 uint8_t sc_link_duplex; 83 }; 84 85 #define DEVNAME(sc) ((sc)->sc_dev.dv_xname) 86 87 #define L1_QUEUE(sc) ((sc)->sc_unit) 88 #define L2_QUEUE(sc) ((sc)->sc_unit) 89 #define L3_QUEUE(sc) ((sc)->sc_unit) 90 #define L4_QUEUE(sc) ((sc)->sc_unit) 91 #define L5_QUEUE(sc) ((sc)->sc_unit) 92 #define DESC_QUEUE(sc) ((sc)->sc_unit) 93 94 #define PORT_FIFO(sc) ((sc)->sc_unit) /* PKO FIFO */ 95 #define PORT_GROUP_RX(sc) ((sc)->sc_unit * 2) /* SSO group for Rx */ 96 #define PORT_GROUP_TX(sc) ((sc)->sc_unit * 2 + 1) /* SSO group for Tx */ 97 #define PORT_MAC(sc) ((sc)->sc_pkomac) 98 #define PORT_PKIND(sc) ((sc)->sc_unit) 99 #define PORT_QPG(sc) ((sc)->sc_unit) 100 #define PORT_STYLE(sc) ((sc)->sc_unit) 101 102 struct ogx_link_ops { 103 const char *link_type; 104 unsigned int link_fifo_speed; /* in Mbps */ 105 /* Initialize link. */ 106 int (*link_init)(struct ogx_softc *); 107 /* Deinitialize link. */ 108 void (*link_down)(struct ogx_softc *); 109 /* Change link parameters. */ 110 void (*link_change)(struct ogx_softc *); 111 /* Query link status. Returns non-zero if status has changed. */ 112 int (*link_status)(struct ogx_softc *); 113 }; 114 115 struct ogx_fifo_group { 116 unsigned int fg_inited; 117 unsigned int fg_speed; 118 }; 119 120 struct ogx_config { 121 unsigned int cfg_nclusters; /* number of parsing clusters */ 122 unsigned int cfg_nfifogrps; /* number of FIFO groups */ 123 unsigned int cfg_nmacs; /* number of MACs */ 124 unsigned int cfg_npqs; /* number of port queues */ 125 unsigned int cfg_npkolvl; /* number of PKO Lx levels */ 126 unsigned int cfg_nullmac; /* index of NULL MAC */ 127 }; 128 129 struct ogx_node { 130 bus_dma_tag_t node_dmat; 131 bus_space_tag_t node_iot; 132 bus_space_handle_t node_fpa3; 133 bus_space_handle_t node_pki; 134 bus_space_handle_t node_pko3; 135 bus_space_handle_t node_sso; 136 137 struct fpa3pool node_pko_pool; 138 struct fpa3pool node_pkt_pool; 139 struct fpa3pool node_sso_pool; 140 struct fpa3aura node_pko_aura; 141 struct fpa3aura node_sso_aura; 142 143 uint64_t node_id; 144 unsigned int node_nclusters; 145 unsigned int node_nunits; 146 struct ogx_fifo_group node_fifogrp[8]; 147 const struct ogx_config *node_cfg; 148 149 struct rwlock node_lock; 150 unsigned int node_flags; 151 #define NODE_INITED 0x01 /* node initialized */ 152 #define NODE_FWREADY 0x02 /* node firmware ready */ 153 }; 154 155 struct ogx_fwhdr { 156 char fw_version[8]; 157 uint64_t fw_size; 158 }; 159 160 #define BGX_PORT_SIZE 0x100000 161 162 #define PORT_RD_8(sc, reg) \ 163 bus_space_read_8((sc)->sc_iot, (sc)->sc_port_ioh, (reg)) 164 #define PORT_WR_8(sc, reg, val) \ 165 bus_space_write_8((sc)->sc_iot, (sc)->sc_port_ioh, (reg), (val)) 166 167 #define NEXUS_RD_8(sc, reg) \ 168 bus_space_read_8((sc)->sc_iot, (sc)->sc_nexus_ioh, (reg)) 169 #define NEXUS_WR_8(sc, reg, val) \ 170 bus_space_write_8((sc)->sc_iot, (sc)->sc_nexus_ioh, (reg), (val)) 171 172 #define FPA3_RD_8(node, reg) \ 173 bus_space_read_8((node)->node_iot, (node)->node_fpa3, (reg)) 174 #define FPA3_WR_8(node, reg, val) \ 175 bus_space_write_8((node)->node_iot, (node)->node_fpa3, (reg), (val)) 176 #define PKI_RD_8(node, reg) \ 177 bus_space_read_8((node)->node_iot, (node)->node_pki, (reg)) 178 #define PKI_WR_8(node, reg, val) \ 179 bus_space_write_8((node)->node_iot, (node)->node_pki, (reg), (val)) 180 #define PKO3_RD_8(node, reg) \ 181 bus_space_read_8((node)->node_iot, (node)->node_pko3, (reg)) 182 #define PKO3_WR_8(node, reg, val) \ 183 bus_space_write_8((node)->node_iot, (node)->node_pko3, (reg), (val)) 184 #define SSO_RD_8(node, reg) \ 185 bus_space_read_8((node)->node_iot, (node)->node_sso, (reg)) 186 #define SSO_WR_8(node, reg, val) \ 187 bus_space_write_8((node)->node_iot, (node)->node_sso, (reg), (val)) 188 189 int ogx_match(struct device *, void *, void *); 190 void ogx_attach(struct device *, struct device *, void *); 191 void ogx_defer(struct device *); 192 193 int ogx_ioctl(struct ifnet *, u_long, caddr_t); 194 void ogx_start(struct ifqueue *); 195 int ogx_send_mbuf(struct ogx_softc *, struct mbuf *); 196 u_int ogx_load_mbufs(struct ogx_softc *, unsigned int); 197 u_int ogx_unload_mbufs(struct ogx_softc *); 198 199 void ogx_media_status(struct ifnet *, struct ifmediareq *); 200 int ogx_media_change(struct ifnet *); 201 int ogx_mii_readreg(struct device *, int, int); 202 void ogx_mii_writereg(struct device *, int, int, int); 203 void ogx_mii_statchg(struct device *); 204 205 int ogx_init(struct ogx_softc *); 206 void ogx_down(struct ogx_softc *); 207 void ogx_iff(struct ogx_softc *); 208 void ogx_rxrefill(void *); 209 int ogx_rxintr(void *); 210 int ogx_txintr(void *); 211 void ogx_tick(void *); 212 213 int ogx_node_init(struct ogx_node **, bus_dma_tag_t, bus_space_tag_t); 214 int ogx_node_load_firmware(struct ogx_node *); 215 void ogx_fpa3_aura_init(struct ogx_node *, struct fpa3aura *, uint32_t, 216 struct fpa3pool *); 217 void ogx_fpa3_aura_load(struct ogx_node *, struct fpa3aura *, size_t, 218 size_t); 219 paddr_t ogx_fpa3_alloc(struct fpa3aura *); 220 void ogx_fpa3_free(struct fpa3aura *, paddr_t); 221 void ogx_fpa3_pool_init(struct ogx_node *, struct fpa3pool *, uint32_t, 222 uint32_t); 223 224 int ogx_sgmii_link_init(struct ogx_softc *); 225 void ogx_sgmii_link_down(struct ogx_softc *); 226 void ogx_sgmii_link_change(struct ogx_softc *); 227 int ogx_sgmii_link_status(struct ogx_softc *); 228 229 static inline paddr_t 230 ogx_kvtophys(vaddr_t kva) 231 { 232 KASSERT(IS_XKPHYS(kva)); 233 return XKPHYS_TO_PHYS(kva); 234 } 235 #define KVTOPHYS(addr) ogx_kvtophys((vaddr_t)(addr)) 236 237 const struct cfattach ogx_ca = { 238 sizeof(struct ogx_softc), ogx_match, ogx_attach 239 }; 240 241 struct cfdriver ogx_cd = { 242 NULL, "ogx", DV_DULL 243 }; 244 245 const struct ogx_config ogx_cn73xx_config = { 246 .cfg_nclusters = 2, 247 .cfg_nfifogrps = 4, 248 .cfg_nmacs = 14, 249 .cfg_npqs = 16, 250 .cfg_npkolvl = 3, 251 .cfg_nullmac = 15, 252 }; 253 254 const struct ogx_config ogx_cn78xx_config = { 255 .cfg_nclusters = 4, 256 .cfg_nfifogrps = 8, 257 .cfg_nmacs = 28, 258 .cfg_npqs = 32, 259 .cfg_npkolvl = 5, 260 .cfg_nullmac = 28, 261 }; 262 263 const struct ogx_link_ops ogx_sgmii_link_ops = { 264 .link_type = "SGMII", 265 .link_fifo_speed = 1000, 266 .link_init = ogx_sgmii_link_init, 267 .link_down = ogx_sgmii_link_down, 268 .link_change = ogx_sgmii_link_change, 269 }; 270 271 const struct ogx_link_ops ogx_xfi_link_ops = { 272 .link_type = "XFI", 273 .link_fifo_speed = 10000, 274 }; 275 276 #define BELTYPE_NONE 0x00 277 #define BELTYPE_MISC 0x01 278 #define BELTYPE_IPv4 0x02 279 #define BELTYPE_IPv6 0x03 280 #define BELTYPE_TCP 0x04 281 #define BELTYPE_UDP 0x05 282 283 static const unsigned int ogx_ltypes[] = { 284 BELTYPE_NONE, /* 0x00 */ 285 BELTYPE_MISC, /* 0x01 Ethernet */ 286 BELTYPE_MISC, /* 0x02 VLAN */ 287 BELTYPE_NONE, /* 0x03 */ 288 BELTYPE_NONE, /* 0x04 */ 289 BELTYPE_MISC, /* 0x05 SNAP */ 290 BELTYPE_MISC, /* 0x06 ARP */ 291 BELTYPE_MISC, /* 0x07 RARP */ 292 BELTYPE_IPv4, /* 0x08 IPv4 */ 293 BELTYPE_IPv4, /* 0x09 IPv4 options */ 294 BELTYPE_IPv6, /* 0x0a IPv6 */ 295 BELTYPE_IPv6, /* 0x0b IPv6 options */ 296 BELTYPE_MISC, /* 0x0c ESP */ 297 BELTYPE_MISC, /* 0x0d IP fragment */ 298 BELTYPE_MISC, /* 0x0e IPcomp */ 299 BELTYPE_NONE, /* 0x0f */ 300 BELTYPE_TCP, /* 0x10 TCP */ 301 BELTYPE_UDP, /* 0x11 UDP */ 302 BELTYPE_MISC, /* 0x12 SCTP */ 303 BELTYPE_UDP, /* 0x13 UDP VXLAN */ 304 BELTYPE_MISC, /* 0x14 GRE */ 305 BELTYPE_MISC, /* 0x15 NVGRE */ 306 BELTYPE_MISC, /* 0x16 GTP */ 307 BELTYPE_UDP, /* 0x17 UDP Geneve */ 308 BELTYPE_NONE, /* 0x18 */ 309 BELTYPE_NONE, /* 0x19 */ 310 BELTYPE_NONE, /* 0x1a */ 311 BELTYPE_NONE, /* 0x1b */ 312 BELTYPE_MISC, /* 0x1c software */ 313 BELTYPE_MISC, /* 0x1d software */ 314 BELTYPE_MISC, /* 0x1e software */ 315 BELTYPE_MISC /* 0x1f software */ 316 }; 317 318 #define OGX_POOL_SSO 0 319 #define OGX_POOL_PKO 1 320 #define OGX_POOL_PKT 2 321 322 #define OGX_AURA_SSO 0 323 #define OGX_AURA_PKO 1 324 #define OGX_AURA_PKT(sc) ((sc)->sc_unit + 2) 325 326 struct ogx_node ogx_node; 327 328 int 329 ogx_match(struct device *parent, void *match, void *aux) 330 { 331 return 1; 332 } 333 334 void 335 ogx_attach(struct device *parent, struct device *self, void *aux) 336 { 337 const struct ogx_config *cfg; 338 struct ogx_fifo_group *fifogrp; 339 struct ogx_node *node; 340 struct ogx_attach_args *oaa = aux; 341 struct ogx_softc *sc = (struct ogx_softc *)self; 342 struct ifnet *ifp = &sc->sc_ac.ac_if; 343 uint64_t lmac_type, lut_index, val; 344 uint32_t lmac; 345 int fgindex = PORT_FIFO(sc) >> 2; 346 int cl, phy_addr, phy_handle; 347 348 if (ogx_node_init(&node, oaa->oaa_dmat, oaa->oaa_iot)) { 349 printf(": node init failed\n"); 350 return; 351 } 352 cfg = node->node_cfg; 353 354 sc->sc_node = node; 355 sc->sc_unit = node->node_nunits++; 356 357 phy_handle = OF_getpropint(oaa->oaa_node, "phy-handle", 0); 358 if (phy_handle == 0) { 359 printf(": no phy-handle\n"); 360 return; 361 } 362 if (cn30xxsmi_get_phy(phy_handle, 0, &sc->sc_smi, &phy_addr)) { 363 printf(": no phy found\n"); 364 return; 365 } 366 367 lmac = OF_getpropint(oaa->oaa_node, "reg", UINT32_MAX); 368 if (lmac == UINT32_MAX) { 369 printf(": no reg property\n"); 370 return; 371 } 372 373 sc->sc_bgxid = oaa->oaa_bgxid; 374 sc->sc_lmacid = lmac; 375 sc->sc_ipdport = sc->sc_bgxid * 0x100 + lmac * 0x10 + 0x800; 376 sc->sc_pkomac = sc->sc_bgxid * 4 + lmac + 2; 377 378 if (OF_getproplen(oaa->oaa_node, "local-mac-address") != 379 ETHER_ADDR_LEN) { 380 printf(": no MAC address\n"); 381 return; 382 } 383 OF_getprop(oaa->oaa_node, "local-mac-address", sc->sc_ac.ac_enaddr, 384 ETHER_ADDR_LEN); 385 386 sc->sc_iot = oaa->oaa_iot; 387 sc->sc_nexus_ioh = oaa->oaa_ioh; 388 if (bus_space_subregion(sc->sc_iot, oaa->oaa_ioh, 389 sc->sc_lmacid * BGX_PORT_SIZE, BGX_PORT_SIZE, &sc->sc_port_ioh)) { 390 printf(": can't map IO subregion\n"); 391 return; 392 } 393 394 val = PORT_RD_8(sc, BGX_CMR_RX_ID_MAP); 395 val &= ~BGX_CMR_RX_ID_MAP_RID_M; 396 val &= ~BGX_CMR_RX_ID_MAP_PKND_M; 397 val |= (uint64_t)(sc->sc_bgxid * 4 + 2 + sc->sc_lmacid) << 398 BGX_CMR_RX_ID_MAP_RID_S; 399 val |= (uint64_t)PORT_PKIND(sc) << BGX_CMR_RX_ID_MAP_PKND_S; 400 PORT_WR_8(sc, BGX_CMR_RX_ID_MAP, val); 401 402 val = PORT_RD_8(sc, BGX_CMR_CHAN_MSK_AND); 403 val |= 0xffffULL << (sc->sc_lmacid * 16); 404 PORT_WR_8(sc, BGX_CMR_CHAN_MSK_AND, val); 405 406 val = PORT_RD_8(sc, BGX_CMR_CHAN_MSK_OR); 407 val |= 0xffffULL << (sc->sc_lmacid * 16); 408 PORT_WR_8(sc, BGX_CMR_CHAN_MSK_OR, val); 409 410 sc->sc_rx_ih = octeon_intr_establish(0x61000 | PORT_GROUP_RX(sc), 411 IPL_NET | IPL_MPSAFE, ogx_rxintr, sc, DEVNAME(sc)); 412 if (sc->sc_rx_ih == NULL) { 413 printf(": could not establish Rx interrupt\n"); 414 return; 415 } 416 sc->sc_tx_ih = octeon_intr_establish(0x61000 | PORT_GROUP_TX(sc), 417 IPL_NET | IPL_MPSAFE, ogx_txintr, sc, DEVNAME(sc)); 418 if (sc->sc_tx_ih == NULL) { 419 printf(": could not establish Tx interrupt\n"); 420 return; 421 } 422 423 val = PORT_RD_8(sc, BGX_CMR_CONFIG); 424 lmac_type = (val & BGX_CMR_CONFIG_LMAC_TYPE_M) >> 425 BGX_CMR_CONFIG_LMAC_TYPE_S; 426 switch (lmac_type) { 427 case 0: 428 sc->sc_link_ops = &ogx_sgmii_link_ops; 429 break; 430 default: 431 printf(": unhandled LMAC type %llu\n", lmac_type); 432 return; 433 } 434 printf(": %s", sc->sc_link_ops->link_type); 435 436 printf(", address %s", ether_sprintf(sc->sc_ac.ac_enaddr)); 437 438 ogx_fpa3_aura_init(node, &sc->sc_pkt_aura, OGX_AURA_PKT(sc), 439 &node->node_pkt_pool); 440 441 sc->sc_rxused = 128; 442 sc->sc_txfree = 128; 443 444 timeout_set(&sc->sc_rxrefill, ogx_rxrefill, sc); 445 timeout_set(&sc->sc_tick, ogx_tick, sc); 446 447 printf("\n"); 448 449 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 450 ifp->if_softc = sc; 451 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 452 ifp->if_xflags |= IFXF_MPSAFE; 453 ifp->if_ioctl = ogx_ioctl; 454 ifp->if_qstart = ogx_start; 455 456 sc->sc_mii.mii_ifp = ifp; 457 sc->sc_mii.mii_readreg = ogx_mii_readreg; 458 sc->sc_mii.mii_writereg = ogx_mii_writereg; 459 sc->sc_mii.mii_statchg = ogx_mii_statchg; 460 ifmedia_init(&sc->sc_media, 0, ogx_media_change, ogx_media_status); 461 462 mii_attach(&sc->sc_dev, &sc->sc_mii, 0xffffffff, phy_addr, 463 MII_OFFSET_ANY, MIIF_NOISOLATE); 464 if (LIST_EMPTY(&sc->sc_mii.mii_phys)) { 465 printf("%s: no PHY found\n", DEVNAME(sc)); 466 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_MANUAL, 0, NULL); 467 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_MANUAL); 468 } else { 469 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 470 471 timeout_add_sec(&sc->sc_tick, 1); 472 } 473 474 /* 475 * Set up the PKI for this port. 476 */ 477 478 val = (uint64_t)PORT_GROUP_RX(sc) << PKI_QPG_TBL_GRP_OK_S; 479 val |= (uint64_t)PORT_GROUP_RX(sc) << PKI_QPG_TBL_GRP_BAD_S; 480 val |= OGX_AURA_PKT(sc) << PKI_QPG_TBL_LAURA_S; 481 PKI_WR_8(node, PKI_QPG_TBL(PORT_QPG(sc)), val); 482 483 for (cl = 0; cl < cfg->cfg_nclusters; cl++) { 484 val = (uint64_t)PORT_QPG(sc) << PKI_CL_STYLE_CFG_QPG_BASE_S; 485 PKI_WR_8(node, PKI_CL_STYLE_CFG(cl, PORT_STYLE(sc)), val); 486 PKI_WR_8(node, PKI_CL_STYLE_CFG2(cl, PORT_STYLE(sc)), 0); 487 PKI_WR_8(node, PKI_CL_STYLE_ALG(cl, PORT_STYLE(sc)), 1u << 31); 488 489 val = PKI_RD_8(node, PKI_CL_PKIND_STYLE(cl, PORT_PKIND(sc))); 490 val &= ~PKI_CL_PKIND_STYLE_PM_M; 491 val &= ~PKI_CL_PKIND_STYLE_STYLE_M; 492 val |= PORT_STYLE(sc) << PKI_CL_PKIND_STYLE_STYLE_S; 493 PKI_WR_8(node, PKI_CL_PKIND_STYLE(cl, PORT_PKIND(sc)), val); 494 } 495 496 val = 5ULL << PKI_STYLE_BUF_FIRST_SKIP_S; 497 val |= ((MCLBYTES - CACHELINESIZE) / sizeof(uint64_t)) << 498 PKI_STYLE_BUF_MB_SIZE_S; 499 PKI_WR_8(node, PKI_STYLE_BUF(PORT_STYLE(sc)), val); 500 501 /* 502 * Set up output queues from the descriptor queue to the port queue. 503 * 504 * The hardware implements a multilevel hierarchy of queues 505 * with configurable priorities. 506 * This driver uses a simple topology where there is one queue 507 * on each level. 508 * 509 * CN73xx: DQ -> L3 -> L2 -> port 510 * CN78xx: DQ -> L5 -> L4 -> L3 -> L2 -> port 511 */ 512 513 /* Map channel to queue L2. */ 514 val = PKO3_RD_8(node, PKO3_L3_L2_SQ_CHANNEL(L2_QUEUE(sc))); 515 val &= ~PKO3_L3_L2_SQ_CHANNEL_CC_ENABLE; 516 val &= ~PKO3_L3_L2_SQ_CHANNEL_M; 517 val |= (uint64_t)sc->sc_ipdport << PKO3_L3_L2_SQ_CHANNEL_S; 518 PKO3_WR_8(node, PKO3_L3_L2_SQ_CHANNEL(L2_QUEUE(sc)), val); 519 520 val = PKO3_RD_8(node, PKO3_MAC_CFG(PORT_MAC(sc))); 521 val &= ~PKO3_MAC_CFG_MIN_PAD_ENA; 522 val &= ~PKO3_MAC_CFG_FCS_ENA; 523 val &= ~PKO3_MAC_CFG_FCS_SOP_OFF_M; 524 val &= ~PKO3_MAC_CFG_FIFO_NUM_M; 525 val |= PORT_FIFO(sc) << PKO3_MAC_CFG_FIFO_NUM_S; 526 PKO3_WR_8(node, PKO3_MAC_CFG(PORT_MAC(sc)), val); 527 528 val = PKO3_RD_8(node, PKO3_MAC_CFG(PORT_MAC(sc))); 529 val &= ~PKO3_MAC_CFG_SKID_MAX_CNT_M; 530 PKO3_WR_8(node, PKO3_MAC_CFG(PORT_MAC(sc)), val); 531 532 PKO3_WR_8(node, PKO3_MCI0_MAX_CRED(PORT_MAC(sc)), 0); 533 PKO3_WR_8(node, PKO3_MCI1_MAX_CRED(PORT_MAC(sc)), 2560 / 16); 534 535 /* Map the port queue to the MAC. */ 536 537 val = (uint64_t)PORT_MAC(sc) << PKO3_L1_SQ_TOPOLOGY_LINK_S; 538 PKO3_WR_8(node, PKO3_L1_SQ_TOPOLOGY(L1_QUEUE(sc)), val); 539 540 val = (uint64_t)PORT_MAC(sc) << PKO3_L1_SQ_SHAPE_LINK_S; 541 PKO3_WR_8(node, PKO3_L1_SQ_SHAPE(L1_QUEUE(sc)), val); 542 543 val = (uint64_t)PORT_MAC(sc) << PKO3_L1_SQ_LINK_LINK_S; 544 PKO3_WR_8(node, PKO3_L1_SQ_LINK(L1_QUEUE(sc)), val); 545 546 /* L1 / port queue */ 547 548 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 549 PKO3_WR_8(node, PKO3_L1_SQ_SCHEDULE(L1_QUEUE(sc)), val); 550 551 val = PKO3_RD_8(node, PKO3_L1_SQ_TOPOLOGY(L1_QUEUE(sc))); 552 val &= ~PKO3_L1_SQ_TOPOLOGY_PRIO_ANCHOR_M; 553 val &= ~PKO3_L1_SQ_TOPOLOGY_RR_PRIO_M; 554 val |= (uint64_t)L2_QUEUE(sc) << PKO3_L1_SQ_TOPOLOGY_PRIO_ANCHOR_S; 555 val |= (uint64_t)0xf << PKO3_L1_SQ_TOPOLOGY_RR_PRIO_S; 556 PKO3_WR_8(node, PKO3_L1_SQ_TOPOLOGY(L1_QUEUE(sc)), val); 557 558 /* L2 */ 559 560 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 561 PKO3_WR_8(node, PKO3_L2_SQ_SCHEDULE(L2_QUEUE(sc)), val); 562 563 val = PKO3_RD_8(node, PKO3_L2_SQ_TOPOLOGY(L2_QUEUE(sc))); 564 val &= ~PKO3_L2_SQ_TOPOLOGY_PRIO_ANCHOR_M; 565 val &= ~PKO3_L2_SQ_TOPOLOGY_PARENT_M; 566 val &= ~PKO3_L2_SQ_TOPOLOGY_RR_PRIO_M; 567 val |= (uint64_t)L3_QUEUE(sc) << PKO3_L2_SQ_TOPOLOGY_PRIO_ANCHOR_S; 568 val |= (uint64_t)L1_QUEUE(sc) << PKO3_L2_SQ_TOPOLOGY_PARENT_S; 569 val |= (uint64_t)0xf << PKO3_L2_SQ_TOPOLOGY_RR_PRIO_S; 570 PKO3_WR_8(node, PKO3_L2_SQ_TOPOLOGY(L2_QUEUE(sc)), val); 571 572 switch (cfg->cfg_npkolvl) { 573 case 3: 574 /* L3 */ 575 576 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 577 PKO3_WR_8(node, PKO3_L3_SQ_SCHEDULE(L3_QUEUE(sc)), val); 578 579 val = PKO3_RD_8(node, PKO3_L3_SQ_TOPOLOGY(L3_QUEUE(sc))); 580 val &= ~PKO3_L3_SQ_TOPOLOGY_PRIO_ANCHOR_M; 581 val &= ~PKO3_L3_SQ_TOPOLOGY_PARENT_M; 582 val &= ~PKO3_L3_SQ_TOPOLOGY_RR_PRIO_M; 583 val |= (uint64_t)DESC_QUEUE(sc) << 584 PKO3_L3_SQ_TOPOLOGY_PRIO_ANCHOR_S; 585 val |= (uint64_t)L2_QUEUE(sc) << PKO3_L3_SQ_TOPOLOGY_PARENT_S; 586 val |= (uint64_t)0xf << PKO3_L3_SQ_TOPOLOGY_RR_PRIO_S; 587 PKO3_WR_8(node, PKO3_L3_SQ_TOPOLOGY(L3_QUEUE(sc)), val); 588 589 /* Descriptor queue */ 590 591 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 592 PKO3_WR_8(node, PKO3_DQ_SCHEDULE(DESC_QUEUE(sc)), val); 593 594 val = (uint64_t)L3_QUEUE(sc) << PKO3_DQ_TOPOLOGY_PARENT_S; 595 PKO3_WR_8(node, PKO3_DQ_TOPOLOGY(DESC_QUEUE(sc)), val); 596 597 break; 598 599 case 5: 600 /* L3 */ 601 602 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 603 PKO3_WR_8(node, PKO3_L3_SQ_SCHEDULE(L3_QUEUE(sc)), val); 604 605 val = PKO3_RD_8(node, PKO3_L3_SQ_TOPOLOGY(L3_QUEUE(sc))); 606 val &= ~PKO3_L3_SQ_TOPOLOGY_PRIO_ANCHOR_M; 607 val &= ~PKO3_L3_SQ_TOPOLOGY_PARENT_M; 608 val &= ~PKO3_L3_SQ_TOPOLOGY_RR_PRIO_M; 609 val |= (uint64_t)L4_QUEUE(sc) << 610 PKO3_L3_SQ_TOPOLOGY_PRIO_ANCHOR_S; 611 val |= (uint64_t)L2_QUEUE(sc) << PKO3_L3_SQ_TOPOLOGY_PARENT_S; 612 val |= (uint64_t)0xf << PKO3_L3_SQ_TOPOLOGY_RR_PRIO_S; 613 PKO3_WR_8(node, PKO3_L3_SQ_TOPOLOGY(L3_QUEUE(sc)), val); 614 615 /* L4 */ 616 617 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 618 PKO3_WR_8(node, PKO3_L4_SQ_SCHEDULE(L4_QUEUE(sc)), val); 619 620 val = PKO3_RD_8(node, PKO3_L4_SQ_TOPOLOGY(L4_QUEUE(sc))); 621 val &= ~PKO3_L4_SQ_TOPOLOGY_PRIO_ANCHOR_M; 622 val &= ~PKO3_L4_SQ_TOPOLOGY_PARENT_M; 623 val &= ~PKO3_L4_SQ_TOPOLOGY_RR_PRIO_M; 624 val |= (uint64_t)L5_QUEUE(sc) << 625 PKO3_L4_SQ_TOPOLOGY_PRIO_ANCHOR_S; 626 val |= (uint64_t)L3_QUEUE(sc) << PKO3_L4_SQ_TOPOLOGY_PARENT_S; 627 val |= (uint64_t)0xf << PKO3_L4_SQ_TOPOLOGY_RR_PRIO_S; 628 PKO3_WR_8(node, PKO3_L4_SQ_TOPOLOGY(L4_QUEUE(sc)), val); 629 630 /* L5 */ 631 632 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 633 PKO3_WR_8(node, PKO3_L5_SQ_SCHEDULE(L5_QUEUE(sc)), val); 634 635 val = PKO3_RD_8(node, PKO3_L5_SQ_TOPOLOGY(L5_QUEUE(sc))); 636 val &= ~PKO3_L5_SQ_TOPOLOGY_PRIO_ANCHOR_M; 637 val &= ~PKO3_L5_SQ_TOPOLOGY_PARENT_M; 638 val &= ~PKO3_L5_SQ_TOPOLOGY_RR_PRIO_M; 639 val |= (uint64_t)DESC_QUEUE(sc) << 640 PKO3_L5_SQ_TOPOLOGY_PRIO_ANCHOR_S; 641 val |= (uint64_t)L4_QUEUE(sc) << PKO3_L5_SQ_TOPOLOGY_PARENT_S; 642 val |= (uint64_t)0xf << PKO3_L5_SQ_TOPOLOGY_RR_PRIO_S; 643 PKO3_WR_8(node, PKO3_L5_SQ_TOPOLOGY(L5_QUEUE(sc)), val); 644 645 /* Descriptor queue */ 646 647 val = (uint64_t)0x10 << PKO3_LX_SQ_SCHEDULE_RR_QUANTUM_S; 648 PKO3_WR_8(node, PKO3_DQ_SCHEDULE(DESC_QUEUE(sc)), val); 649 650 val = (uint64_t)L5_QUEUE(sc) << PKO3_DQ_TOPOLOGY_PARENT_S; 651 PKO3_WR_8(node, PKO3_DQ_TOPOLOGY(DESC_QUEUE(sc)), val); 652 653 break; 654 655 default: 656 printf(": unhandled number of PKO levels (%u)\n", 657 cfg->cfg_npkolvl); 658 return; 659 } 660 661 /* Descriptor queue, common part */ 662 663 PKO3_WR_8(node, PKO3_DQ_WM_CTL(DESC_QUEUE(sc)), PKO3_DQ_WM_CTL_KIND); 664 665 val = PKO3_RD_8(node, PKO3_PDM_DQ_MINPAD(DESC_QUEUE(sc))); 666 val &= ~PKO3_PDM_DQ_MINPAD_MINPAD; 667 PKO3_WR_8(node, PKO3_PDM_DQ_MINPAD(DESC_QUEUE(sc)), val); 668 669 lut_index = sc->sc_bgxid * 0x40 + lmac * 0x10; 670 val = PKO3_LUT_VALID | (L1_QUEUE(sc) << PKO3_LUT_PQ_IDX_S) | 671 (L2_QUEUE(sc) << PKO3_LUT_QUEUE_NUM_S); 672 PKO3_WR_8(node, PKO3_LUT(lut_index), val); 673 674 fifogrp = &node->node_fifogrp[fgindex]; 675 fifogrp->fg_speed += sc->sc_link_ops->link_fifo_speed; 676 677 /* 678 * Defer the rest of the initialization so that FIFO groups 679 * can be configured properly. 680 */ 681 config_defer(&sc->sc_dev, ogx_defer); 682 } 683 684 void 685 ogx_defer(struct device *dev) 686 { 687 struct ogx_fifo_group *fifogrp; 688 struct ogx_softc *sc = (struct ogx_softc *)dev; 689 struct ogx_node *node = sc->sc_node; 690 struct ifnet *ifp = &sc->sc_ac.ac_if; 691 uint64_t grprate, val; 692 int fgindex = PORT_FIFO(sc) >> 2; 693 694 fifogrp = &node->node_fifogrp[fgindex]; 695 if (fifogrp->fg_inited == 0) { 696 /* Adjust the total rate of the fifo group. */ 697 grprate = 0; 698 while (fifogrp->fg_speed > (6250 << grprate)) 699 grprate++; 700 if (grprate > 5) 701 grprate = 5; 702 703 val = PKO3_RD_8(node, PKO3_PTGF_CFG(fgindex)); 704 val &= ~PKO3_PTGF_CFG_RATE_M; 705 val |= grprate << PKO3_PTGF_CFG_RATE_S; 706 PKO3_WR_8(node, PKO3_PTGF_CFG(fgindex), val); 707 708 fifogrp->fg_inited = 1; 709 } 710 711 if_attach(ifp); 712 ether_ifattach(ifp); 713 } 714 715 int 716 ogx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 717 { 718 struct ogx_softc *sc = ifp->if_softc; 719 struct ifreq *ifr = (struct ifreq *)data; 720 int error = 0; 721 int s; 722 723 s = splnet(); 724 725 switch (cmd) { 726 case SIOCSIFADDR: 727 ifp->if_flags |= IFF_UP; 728 /* FALLTHROUGH */ 729 730 case SIOCSIFFLAGS: 731 if (ISSET(ifp->if_flags, IFF_UP)) { 732 if (ISSET(ifp->if_flags, IFF_RUNNING)) 733 error = ENETRESET; 734 else 735 error = ogx_init(sc); 736 } else { 737 if (ISSET(ifp->if_flags, IFF_RUNNING)) 738 ogx_down(sc); 739 } 740 break; 741 742 case SIOCGIFMEDIA: 743 case SIOCSIFMEDIA: 744 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd); 745 break; 746 747 default: 748 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 749 break; 750 } 751 752 if (error == ENETRESET) { 753 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 754 (IFF_UP | IFF_RUNNING)) 755 ogx_iff(sc); 756 error = 0; 757 } 758 759 splx(s); 760 761 return error; 762 } 763 764 int 765 ogx_init(struct ogx_softc *sc) 766 { 767 struct ogx_node *node = sc->sc_node; 768 struct ifnet *ifp = &sc->sc_ac.ac_if; 769 uint64_t op; 770 int error; 771 772 error = ogx_node_load_firmware(node); 773 if (error != 0) 774 return error; 775 776 ogx_iff(sc); 777 778 SSO_WR_8(sc->sc_node, SSO_GRP_INT_THR(PORT_GROUP_RX(sc)), 1); 779 SSO_WR_8(sc->sc_node, SSO_GRP_INT_THR(PORT_GROUP_TX(sc)), 1); 780 781 sc->sc_link_ops->link_init(sc); 782 if (!LIST_EMPTY(&sc->sc_mii.mii_phys)) 783 mii_mediachg(&sc->sc_mii); 784 785 /* Open the descriptor queue. */ 786 op = PKO3_LD_IO | PKO3_LD_DID; 787 op |= node->node_id << PKO3_LD_NODE_S; 788 op |= PKO3_DQOP_OPEN << PKO3_LD_OP_S; 789 op |= DESC_QUEUE(sc) << PKO3_LD_DQ_S; 790 (void)octeon_xkphys_read_8(op); 791 792 ifp->if_flags |= IFF_RUNNING; 793 ifq_restart(&ifp->if_snd); 794 795 timeout_add(&sc->sc_rxrefill, 1); 796 timeout_add_sec(&sc->sc_tick, 1); 797 798 return 0; 799 } 800 801 void 802 ogx_down(struct ogx_softc *sc) 803 { 804 struct ifnet *ifp = &sc->sc_ac.ac_if; 805 struct ogx_node *node = sc->sc_node; 806 uint64_t op, val; 807 unsigned int nused; 808 809 CLR(ifp->if_flags, IFF_RUNNING); 810 811 /* Drain the descriptor queue. */ 812 val = PKO3_LX_SQ_SW_XOFF_DRAIN; 813 val |= PKO3_LX_SQ_SW_XOFF_DRAIN_NULL_LINK; 814 PKO3_WR_8(node, PKO3_DQ_SW_XOFF(DESC_QUEUE(sc)), val); 815 (void)PKO3_RD_8(node, PKO3_DQ_SW_XOFF(DESC_QUEUE(sc))); 816 817 delay(1000); 818 819 /* Finish the drain operation. */ 820 PKO3_WR_8(node, PKO3_DQ_SW_XOFF(DESC_QUEUE(sc)), 0); 821 (void)PKO3_RD_8(node, PKO3_DQ_SW_XOFF(DESC_QUEUE(sc))); 822 823 /* Close the descriptor queue. */ 824 op = PKO3_LD_IO | PKO3_LD_DID; 825 op |= node->node_id << PKO3_LD_NODE_S; 826 op |= PKO3_DQOP_CLOSE << PKO3_LD_OP_S; 827 op |= DESC_QUEUE(sc) << PKO3_LD_DQ_S; 828 (void)octeon_xkphys_read_8(op); 829 830 /* Disable data transfer. */ 831 val = PORT_RD_8(sc, BGX_CMR_CONFIG); 832 val &= ~BGX_CMR_CONFIG_DATA_PKT_RX_EN; 833 val &= ~BGX_CMR_CONFIG_DATA_PKT_TX_EN; 834 PORT_WR_8(sc, BGX_CMR_CONFIG, val); 835 (void)PORT_RD_8(sc, BGX_CMR_CONFIG); 836 837 if (!LIST_EMPTY(&sc->sc_mii.mii_phys)) 838 mii_down(&sc->sc_mii); 839 sc->sc_link_ops->link_down(sc); 840 841 ifq_clr_oactive(&ifp->if_snd); 842 ifq_barrier(&ifp->if_snd); 843 844 timeout_del_barrier(&sc->sc_rxrefill); 845 timeout_del_barrier(&sc->sc_tick); 846 847 nused = ogx_unload_mbufs(sc); 848 atomic_add_int(&sc->sc_rxused, nused); 849 } 850 851 void 852 ogx_iff(struct ogx_softc *sc) 853 { 854 struct arpcom *ac = &sc->sc_ac; 855 struct ifnet *ifp = &sc->sc_ac.ac_if; 856 struct ether_multi *enm; 857 struct ether_multistep step; 858 uint64_t rx_adr_ctl; 859 uint64_t val; 860 int cidx, clast, i; 861 862 rx_adr_ctl = PORT_RD_8(sc, BGX_CMR_RX_ADR_CTL); 863 rx_adr_ctl |= BGX_CMR_RX_ADR_CTL_BCST_ACCEPT; 864 rx_adr_ctl |= BGX_CMR_RX_ADR_CTL_CAM_ACCEPT; 865 rx_adr_ctl &= ~BGX_CMR_RX_ADR_CTL_MCST_MODE_ALL; 866 ifp->if_flags &= ~IFF_ALLMULTI; 867 868 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 869 ifp->if_flags |= IFF_ALLMULTI; 870 rx_adr_ctl &= ~BGX_CMR_RX_ADR_CTL_CAM_ACCEPT; 871 rx_adr_ctl |= BGX_CMR_RX_ADR_CTL_MCST_MODE_ALL; 872 } else if (ac->ac_multirangecnt > 0 || ac->ac_multicnt >= OGX_NCAM) { 873 ifp->if_flags |= IFF_ALLMULTI; 874 rx_adr_ctl |= BGX_CMR_RX_ADR_CTL_MCST_MODE_ALL; 875 } else { 876 rx_adr_ctl |= BGX_CMR_RX_ADR_CTL_MCST_MODE_CAM; 877 } 878 879 PORT_WR_8(sc, BGX_CMR_RX_ADR_CTL, rx_adr_ctl); 880 881 cidx = sc->sc_lmacid * OGX_NCAM; 882 clast = (sc->sc_lmacid + 1) * OGX_NCAM; 883 884 if (!ISSET(ifp->if_flags, IFF_PROMISC)) { 885 val = BGX_CMR_RX_ADR_CAM_EN | ((uint64_t)sc->sc_lmacid 886 << BGX_CMR_RX_ADR_CAM_ID_S); 887 for (i = 0; i < ETHER_ADDR_LEN; i++) { 888 val |= (uint64_t)ac->ac_enaddr[i] << 889 ((ETHER_ADDR_LEN - 1 - i) * 8); 890 } 891 NEXUS_WR_8(sc, BGX_CMR_RX_ADR_CAM(cidx++), val); 892 } 893 894 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 895 ETHER_FIRST_MULTI(step, ac, enm); 896 while (enm != NULL) { 897 val = BGX_CMR_RX_ADR_CAM_EN | ((uint64_t)sc->sc_lmacid 898 << BGX_CMR_RX_ADR_CAM_ID_S); 899 for (i = 0; i < ETHER_ADDR_LEN; i++) 900 val |= (uint64_t)enm->enm_addrlo[i] << (i * 8); 901 KASSERT(cidx < clast); 902 NEXUS_WR_8(sc, BGX_CMR_RX_ADR_CAM(cidx++), val); 903 904 ETHER_NEXT_MULTI(step, enm); 905 } 906 } 907 908 /* Disable any remaining address CAM entries. */ 909 while (cidx < clast) 910 NEXUS_WR_8(sc, BGX_CMR_RX_ADR_CAM(cidx++), 0); 911 } 912 913 static inline uint64_t * 914 ogx_get_work(struct ogx_node *node, uint32_t group) 915 { 916 uint64_t op, resp; 917 918 op = SSO_LD_IO | SSO_LD_DID; 919 op |= node->node_id << SSO_LD_NODE_S; 920 op |= SSO_LD_GROUPED | (group << SSO_LD_INDEX_S); 921 resp = octeon_xkphys_read_8(op); 922 923 if (resp & SSO_LD_RTN_NO_WORK) 924 return NULL; 925 926 return (uint64_t *)PHYS_TO_XKPHYS(resp & SSO_LD_RTN_ADDR_M, CCA_CACHED); 927 } 928 929 static inline struct mbuf * 930 ogx_extract_mbuf(struct ogx_softc *sc, paddr_t pktbuf) 931 { 932 struct mbuf *m, **pm; 933 934 pm = (struct mbuf **)PHYS_TO_XKPHYS(pktbuf, CCA_CACHED) - 1; 935 m = *pm; 936 *pm = NULL; 937 KASSERTMSG((paddr_t)m->m_pkthdr.ph_cookie == pktbuf, 938 "%s: corrupt packet pool, mbuf cookie %p != pktbuf %p", 939 DEVNAME(sc), m->m_pkthdr.ph_cookie, (void *)pktbuf); 940 m->m_pkthdr.ph_cookie = NULL; 941 return m; 942 } 943 944 void 945 ogx_rxrefill(void *arg) 946 { 947 struct ogx_softc *sc = arg; 948 unsigned int to_alloc; 949 950 if (sc->sc_rxused > 0) { 951 to_alloc = atomic_swap_uint(&sc->sc_rxused, 0); 952 to_alloc = ogx_load_mbufs(sc, to_alloc); 953 if (to_alloc > 0) { 954 atomic_add_int(&sc->sc_rxused, to_alloc); 955 timeout_add(&sc->sc_rxrefill, 1); 956 } 957 } 958 } 959 960 void 961 ogx_tick(void *arg) 962 { 963 struct ogx_softc *sc = arg; 964 int s; 965 966 s = splnet(); 967 if (!LIST_EMPTY(&sc->sc_mii.mii_phys)) { 968 mii_tick(&sc->sc_mii); 969 } else { 970 if (sc->sc_link_ops->link_status(sc)) 971 sc->sc_link_ops->link_change(sc); 972 } 973 splx(s); 974 975 timeout_add_sec(&sc->sc_tick, 1); 976 } 977 978 int 979 ogx_rxintr(void *arg) 980 { 981 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 982 struct mbuf *m, *m0, *mprev; 983 struct ogx_softc *sc = arg; 984 struct ogx_node *node = sc->sc_node; 985 struct ifnet *ifp = &sc->sc_ac.ac_if; 986 paddr_t pktbuf, pktdata; 987 uint64_t *work; 988 uint64_t nsegs; 989 unsigned int rxused = 0; 990 991 /* Acknowledge the interrupt. */ 992 SSO_WR_8(node, SSO_GRP_INT(PORT_GROUP_RX(sc)), SSO_GRP_INT_EXE_INT); 993 994 for (;;) { 995 uint64_t errcode, errlevel; 996 uint64_t word3; 997 size_t pktlen, left; 998 #ifdef DIAGNOSTIC 999 unsigned int pkind; 1000 #endif 1001 1002 work = ogx_get_work(sc->sc_node, PORT_GROUP_RX(sc)); 1003 if (work == NULL) 1004 break; 1005 1006 #ifdef DIAGNOSTIC 1007 pkind = (work[0] & PKI_WORD0_PKIND_M) >> PKI_WORD0_PKIND_S; 1008 if (__predict_false(pkind != PORT_PKIND(sc))) { 1009 printf("%s: unexpected pkind %u, should be %u\n", 1010 DEVNAME(sc), pkind, PORT_PKIND(sc)); 1011 goto wqe_error; 1012 } 1013 #endif 1014 1015 nsegs = (work[0] & PKI_WORD0_BUFS_M) >> PKI_WORD0_BUFS_S; 1016 word3 = work[3]; 1017 1018 errlevel = (work[2] & PKI_WORD2_ERR_LEVEL_M) >> 1019 PKI_WORD2_ERR_LEVEL_S; 1020 errcode = (work[2] & PKI_WORD2_ERR_CODE_M) >> 1021 PKI_WORD2_ERR_CODE_S; 1022 if (__predict_false(errlevel <= 1 && errcode != 0)) { 1023 ifp->if_ierrors++; 1024 goto drop; 1025 } 1026 1027 KASSERT(nsegs > 0); 1028 rxused += nsegs; 1029 1030 pktlen = (work[1] & PKI_WORD1_LEN_M) >> PKI_WORD1_LEN_S; 1031 left = pktlen; 1032 1033 m0 = NULL; 1034 mprev = NULL; 1035 while (nsegs-- > 0) { 1036 size_t size; 1037 1038 pktdata = (word3 & PKI_WORD3_ADDR_M) >> 1039 PKI_WORD3_ADDR_S; 1040 pktbuf = pktdata & ~(CACHELINESIZE - 1); 1041 size = (word3 & PKI_WORD3_SIZE_M) >> PKI_WORD3_SIZE_S; 1042 if (size > left) 1043 size = left; 1044 1045 m = ogx_extract_mbuf(sc, pktbuf); 1046 m->m_data += (pktdata - pktbuf) & (CACHELINESIZE - 1); 1047 m->m_len = size; 1048 left -= size; 1049 1050 /* pktdata can be unaligned. */ 1051 memcpy(&word3, (void *)PHYS_TO_XKPHYS(pktdata - 1052 sizeof(uint64_t), CCA_CACHED), sizeof(uint64_t)); 1053 1054 if (m0 == NULL) { 1055 m0 = m; 1056 } else { 1057 m->m_flags &= ~M_PKTHDR; 1058 mprev->m_next = m; 1059 } 1060 mprev = m; 1061 } 1062 1063 m0->m_pkthdr.len = pktlen; 1064 ml_enqueue(&ml, m0); 1065 1066 continue; 1067 1068 drop: 1069 /* Return the buffers back to the pool. */ 1070 while (nsegs-- > 0) { 1071 pktdata = (word3 & PKI_WORD3_ADDR_M) >> 1072 PKI_WORD3_ADDR_S; 1073 pktbuf = pktdata & ~(CACHELINESIZE - 1); 1074 /* pktdata can be unaligned. */ 1075 memcpy(&word3, (void *)PHYS_TO_XKPHYS(pktdata - 1076 sizeof(uint64_t), CCA_CACHED), sizeof(uint64_t)); 1077 ogx_fpa3_free(&sc->sc_pkt_aura, pktbuf); 1078 } 1079 } 1080 1081 if_input(ifp, &ml); 1082 1083 rxused = ogx_load_mbufs(sc, rxused); 1084 if (rxused != 0) { 1085 atomic_add_int(&sc->sc_rxused, rxused); 1086 timeout_add(&sc->sc_rxrefill, 1); 1087 } 1088 1089 return 1; 1090 1091 #ifdef DIAGNOSTIC 1092 wqe_error: 1093 printf("work0: %016llx\n", work[0]); 1094 printf("work1: %016llx\n", work[1]); 1095 printf("work2: %016llx\n", work[2]); 1096 printf("work3: %016llx\n", work[3]); 1097 printf("work4: %016llx\n", work[4]); 1098 panic("%s: %s: wqe error", DEVNAME(sc), __func__); 1099 #endif 1100 } 1101 1102 int 1103 ogx_txintr(void *arg) 1104 { 1105 struct ogx_softc *sc = arg; 1106 struct ogx_node *node = sc->sc_node; 1107 struct ifnet *ifp = &sc->sc_ac.ac_if; 1108 struct mbuf *m; 1109 uint64_t *work; 1110 unsigned int nfreed = 0; 1111 1112 /* Acknowledge the interrupt. */ 1113 SSO_WR_8(node, SSO_GRP_INT(PORT_GROUP_TX(sc)), SSO_GRP_INT_EXE_INT); 1114 1115 for (;;) { 1116 work = ogx_get_work(node, PORT_GROUP_TX(sc)); 1117 if (work == NULL) 1118 break; 1119 1120 /* 1121 * work points to ph_cookie via the xkphys segment. 1122 * ph_cookie contains the original mbuf pointer. 1123 */ 1124 m = *(struct mbuf **)work; 1125 KASSERT(m->m_pkthdr.ph_ifidx == (u_int)(uintptr_t)sc); 1126 m->m_pkthdr.ph_ifidx = 0; 1127 m_freem(m); 1128 nfreed++; 1129 } 1130 1131 if (nfreed > 0 && atomic_add_int_nv(&sc->sc_txfree, nfreed) == nfreed) 1132 ifq_restart(&ifp->if_snd); 1133 1134 return 1; 1135 } 1136 1137 unsigned int 1138 ogx_load_mbufs(struct ogx_softc *sc, unsigned int n) 1139 { 1140 struct mbuf *m; 1141 paddr_t pktbuf; 1142 1143 for ( ; n > 0; n--) { 1144 m = MCLGETI(NULL, M_NOWAIT, NULL, MCLBYTES); 1145 if (m == NULL) 1146 break; 1147 1148 m->m_data = (void *)(((vaddr_t)m->m_data + CACHELINESIZE) & 1149 ~(CACHELINESIZE - 1)); 1150 ((struct mbuf **)m->m_data)[-1] = m; 1151 1152 pktbuf = KVTOPHYS(m->m_data); 1153 m->m_pkthdr.ph_cookie = (void *)pktbuf; 1154 ogx_fpa3_free(&sc->sc_pkt_aura, pktbuf); 1155 } 1156 return n; 1157 } 1158 1159 unsigned int 1160 ogx_unload_mbufs(struct ogx_softc *sc) 1161 { 1162 struct mbuf *m; 1163 paddr_t pktbuf; 1164 unsigned int n = 0; 1165 1166 for (;;) { 1167 pktbuf = ogx_fpa3_alloc(&sc->sc_pkt_aura); 1168 if (pktbuf == 0) 1169 break; 1170 m = ogx_extract_mbuf(sc, pktbuf); 1171 m_freem(m); 1172 n++; 1173 } 1174 return n; 1175 } 1176 1177 void 1178 ogx_start(struct ifqueue *ifq) 1179 { 1180 struct ifnet *ifp = ifq->ifq_if; 1181 struct ogx_softc *sc = ifp->if_softc; 1182 struct mbuf *m; 1183 unsigned int txfree, txused; 1184 1185 txfree = READ_ONCE(sc->sc_txfree); 1186 txused = 0; 1187 1188 while (txused < txfree) { 1189 m = ifq_dequeue(ifq); 1190 if (m == NULL) 1191 break; 1192 1193 #if NBPFILTER > 0 1194 if (ifp->if_bpf != NULL) 1195 bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT); 1196 #endif 1197 1198 if (ogx_send_mbuf(sc, m) != 0) { 1199 m_freem(m); 1200 ifp->if_oerrors++; 1201 continue; 1202 } 1203 txused++; 1204 } 1205 1206 if (atomic_sub_int_nv(&sc->sc_txfree, txused) == 0) 1207 ifq_set_oactive(ifq); 1208 } 1209 1210 int 1211 ogx_send_mbuf(struct ogx_softc *sc, struct mbuf *m0) 1212 { 1213 struct mbuf *m; 1214 uint64_t scroff, word; 1215 unsigned int nfrags; 1216 1217 /* Save original pointer for freeing after transmission. */ 1218 m0->m_pkthdr.ph_cookie = m0; 1219 /* Add a tag for sanity checking. */ 1220 m0->m_pkthdr.ph_ifidx = (u_int)(uintptr_t)sc; 1221 1222 /* Flush pending writes before packet submission. */ 1223 octeon_syncw(); 1224 1225 /* Block until any previous LMTDMA request has been processed. */ 1226 octeon_synciobdma(); 1227 1228 /* Get the LMTDMA region offset in the scratchpad. */ 1229 scroff = 2 * 0x80; 1230 1231 word = PKO3_SEND_HDR_DF; 1232 word |= m0->m_pkthdr.len << PKO3_SEND_HDR_TOTAL_S; 1233 octeon_cvmseg_write_8(scroff, word); 1234 scroff += sizeof(word); 1235 1236 for (m = m0, nfrags = 0; m != NULL && nfrags < 13; 1237 m = m->m_next, nfrags++) { 1238 word = PKO3_SUBDC3_SEND_GATHER << PKO3_SUBC_BUF_PTR_SUBDC3_S; 1239 word |= KVTOPHYS(m->m_data) << PKO3_SUBC_BUF_PTR_ADDR_S; 1240 word |= (uint64_t)m->m_len << PKO3_SUBC_BUF_PTR_SIZE_S; 1241 octeon_cvmseg_write_8(scroff, word); 1242 scroff += sizeof(word); 1243 } 1244 1245 if (m != NULL) { 1246 if (m_defrag(m0, M_DONTWAIT) != 0) 1247 return ENOMEM; 1248 1249 /* Discard previously set fragments. */ 1250 scroff -= sizeof(word) * nfrags; 1251 1252 word = PKO3_SUBDC3_SEND_GATHER << PKO3_SUBC_BUF_PTR_SUBDC3_S; 1253 word |= KVTOPHYS(m0->m_data) << PKO3_SUBC_BUF_PTR_ADDR_S; 1254 word |= (uint64_t)m0->m_len << PKO3_SUBC_BUF_PTR_SIZE_S; 1255 octeon_cvmseg_write_8(scroff, word); 1256 scroff += sizeof(word); 1257 } 1258 1259 /* Send work when ready to free the mbuf. */ 1260 word = PKO3_SEND_WORK_CODE << PKO3_SEND_SUBDC4_CODE_S; 1261 word |= KVTOPHYS(&m0->m_pkthdr.ph_cookie) << PKO3_SEND_WORK_ADDR_S; 1262 word |= (uint64_t)PORT_GROUP_TX(sc) << PKO3_SEND_WORK_GRP_S; 1263 word |= 2ULL << PKO3_SEND_WORK_TT_S; 1264 octeon_cvmseg_write_8(scroff, word); 1265 scroff += sizeof(word); 1266 1267 /* Submit the command. */ 1268 word = PKO3_LMTDMA_DID; 1269 word |= ((2ULL * 0x80) >> 3) << PKO3_LMTDMA_SCRADDR_S; 1270 word |= 1ULL << PKO3_LMTDMA_RTNLEN_S; 1271 word |= DESC_QUEUE(sc) << PKO3_LMTDMA_DQ_S; 1272 octeon_lmtdma_write_8((scroff - 8) & 0x78, word); 1273 1274 return 0; 1275 } 1276 1277 int 1278 ogx_media_change(struct ifnet *ifp) 1279 { 1280 struct ogx_softc *sc = ifp->if_softc; 1281 1282 if (!LIST_EMPTY(&sc->sc_mii.mii_phys)) 1283 mii_mediachg(&sc->sc_mii); 1284 1285 return 0; 1286 } 1287 1288 void 1289 ogx_media_status(struct ifnet *ifp, struct ifmediareq *imr) 1290 { 1291 struct ogx_softc *sc = ifp->if_softc; 1292 1293 if (!LIST_EMPTY(&sc->sc_mii.mii_phys)) { 1294 mii_pollstat(&sc->sc_mii); 1295 imr->ifm_status = sc->sc_mii.mii_media_status; 1296 imr->ifm_active = sc->sc_mii.mii_media_active; 1297 } 1298 } 1299 1300 int 1301 ogx_mii_readreg(struct device *self, int phy_no, int reg) 1302 { 1303 struct ogx_softc *sc = (struct ogx_softc *)self; 1304 1305 return cn30xxsmi_read(sc->sc_smi, phy_no, reg); 1306 } 1307 1308 void 1309 ogx_mii_writereg(struct device *self, int phy_no, int reg, int value) 1310 { 1311 struct ogx_softc *sc = (struct ogx_softc *)self; 1312 1313 cn30xxsmi_write(sc->sc_smi, phy_no, reg, value); 1314 } 1315 1316 void 1317 ogx_mii_statchg(struct device *self) 1318 { 1319 struct ogx_softc *sc = (struct ogx_softc *)self; 1320 1321 if (ISSET(sc->sc_mii.mii_media_active, IFM_FDX)) 1322 sc->sc_link_duplex = 1; 1323 else 1324 sc->sc_link_duplex = 0; 1325 sc->sc_link_ops->link_change(sc); 1326 } 1327 1328 int 1329 ogx_sgmii_link_init(struct ogx_softc *sc) 1330 { 1331 uint64_t cpu_freq = octeon_boot_info->eclock / 1000000; 1332 uint64_t val; 1333 int align = 1; 1334 1335 val = PORT_RD_8(sc, BGX_GMP_GMI_TX_APPEND); 1336 val |= BGX_GMP_GMI_TX_APPEND_FCS; 1337 val |= BGX_GMP_GMI_TX_APPEND_PAD; 1338 if (ISSET(val, BGX_GMP_GMI_TX_APPEND_PREAMBLE)) 1339 align = 0; 1340 PORT_WR_8(sc, BGX_GMP_GMI_TX_APPEND, val); 1341 PORT_WR_8(sc, BGX_GMP_GMI_TX_MIN_PKT, 59); 1342 PORT_WR_8(sc, BGX_GMP_GMI_TX_THRESH, 0x20); 1343 1344 val = PORT_RD_8(sc, BGX_GMP_GMI_TX_SGMII_CTL); 1345 if (align) 1346 val |= BGX_GMP_GMI_TX_SGMII_CTL_ALIGN; 1347 else 1348 val &= ~BGX_GMP_GMI_TX_SGMII_CTL_ALIGN; 1349 PORT_WR_8(sc, BGX_GMP_GMI_TX_SGMII_CTL, val); 1350 1351 /* Set timing for SGMII. */ 1352 val = PORT_RD_8(sc, BGX_GMP_PCS_LINK_TIMER); 1353 val &= ~BGX_GMP_PCS_LINK_TIMER_COUNT_M; 1354 val |= (1600 * cpu_freq) >> 10; 1355 PORT_WR_8(sc, BGX_GMP_PCS_LINK_TIMER, val); 1356 1357 return 0; 1358 } 1359 1360 void 1361 ogx_sgmii_link_down(struct ogx_softc *sc) 1362 { 1363 uint64_t val; 1364 int timeout; 1365 1366 /* Wait until the port is idle. */ 1367 for (timeout = 1000; timeout > 0; timeout--) { 1368 const uint64_t idlemask = BGX_GMP_GMI_PRT_CFG_RX_IDLE | 1369 BGX_GMP_GMI_PRT_CFG_TX_IDLE; 1370 val = PORT_RD_8(sc, BGX_GMP_GMI_PRT_CFG); 1371 if ((val & idlemask) == idlemask) 1372 break; 1373 delay(1000); 1374 } 1375 if (timeout == 0) 1376 printf("%s: port idle timeout\n", DEVNAME(sc)); 1377 1378 /* Disable autonegotiation and power down the link. */ 1379 val = PORT_RD_8(sc, BGX_GMP_PCS_MR_CONTROL); 1380 val &= ~BGX_GMP_PCS_MR_CONTROL_AN_EN; 1381 val |= BGX_GMP_PCS_MR_CONTROL_PWR_DN; 1382 PORT_WR_8(sc, BGX_GMP_PCS_MR_CONTROL, val); 1383 } 1384 1385 void 1386 ogx_sgmii_link_change(struct ogx_softc *sc) 1387 { 1388 struct ifnet *ifp = &sc->sc_ac.ac_if; 1389 uint64_t config; 1390 uint64_t misc_ctl; 1391 uint64_t prt_cfg = 0; 1392 uint64_t samp_pt; 1393 uint64_t tx_burst, tx_slot; 1394 uint64_t val; 1395 int timeout; 1396 1397 if (!LINK_STATE_IS_UP(ifp->if_link_state)) { 1398 misc_ctl = PORT_RD_8(sc, BGX_GMP_PCS_MISC_CTL); 1399 misc_ctl |= BGX_GMP_PCS_MISC_CTL_GMXENO; 1400 PORT_WR_8(sc, BGX_GMP_PCS_MISC_CTL, misc_ctl); 1401 return; 1402 } 1403 1404 val = PORT_RD_8(sc, BGX_CMR_CONFIG); 1405 val |= BGX_CMR_CONFIG_ENABLE; 1406 PORT_WR_8(sc, BGX_CMR_CONFIG, val); 1407 1408 /* Reset the PCS. */ 1409 val = PORT_RD_8(sc, BGX_GMP_PCS_MR_CONTROL); 1410 val |= BGX_GMP_PCS_MR_CONTROL_RESET; 1411 PORT_WR_8(sc, BGX_GMP_PCS_MR_CONTROL_RESET, val); 1412 1413 /* Wait for the reset to complete. */ 1414 timeout = 100000; 1415 while (timeout-- > 0) { 1416 val = PORT_RD_8(sc, BGX_GMP_PCS_MR_CONTROL); 1417 if (!ISSET(val, BGX_GMP_PCS_MR_CONTROL_RESET)) 1418 break; 1419 delay(10); 1420 } 1421 if (timeout == 0) 1422 printf("%s: SGMII reset timeout\n", DEVNAME(sc)); 1423 1424 /* Use MAC mode. */ 1425 val = PORT_RD_8(sc, BGX_GMP_PCS_MISC_CTL); 1426 val &= ~BGX_GMP_PCS_MISC_CTL_MAC_PHY; 1427 val &= ~BGX_GMP_PCS_MISC_CTL_MODE; 1428 PORT_WR_8(sc, BGX_GMP_PCS_MISC_CTL, val); 1429 1430 /* Start autonegotiation between the SoC and the PHY. */ 1431 val = PORT_RD_8(sc, BGX_GMP_PCS_MR_CONTROL); 1432 val |= BGX_GMP_PCS_MR_CONTROL_AN_EN; 1433 val |= BGX_GMP_PCS_MR_CONTROL_RST_AN; 1434 val &= ~BGX_GMP_PCS_MR_CONTROL_PWR_DN; 1435 PORT_WR_8(sc, BGX_GMP_PCS_MR_CONTROL, val); 1436 1437 /* Wait for the autonegotiation to complete. */ 1438 timeout = 100000; 1439 while (timeout-- > 0) { 1440 val = PORT_RD_8(sc, BGX_GMP_PCS_MR_STATUS); 1441 if (ISSET(val, BGX_GMP_PCS_MR_STATUS_AN_CPT)) 1442 break; 1443 delay(10); 1444 } 1445 if (timeout == 0) 1446 printf("%s: SGMII autonegotiation timeout\n", DEVNAME(sc)); 1447 1448 /* Stop Rx and Tx engines. */ 1449 config = PORT_RD_8(sc, BGX_CMR_CONFIG); 1450 config &= ~BGX_CMR_CONFIG_DATA_PKT_RX_EN; 1451 config &= ~BGX_CMR_CONFIG_DATA_PKT_TX_EN; 1452 PORT_WR_8(sc, BGX_CMR_CONFIG, config); 1453 (void)PORT_RD_8(sc, BGX_CMR_CONFIG); 1454 1455 /* Wait until the engines are idle. */ 1456 for (timeout = 1000000; timeout > 0; timeout--) { 1457 const uint64_t idlemask = BGX_GMP_GMI_PRT_CFG_RX_IDLE | 1458 BGX_GMP_GMI_PRT_CFG_TX_IDLE; 1459 prt_cfg = PORT_RD_8(sc, BGX_GMP_GMI_PRT_CFG); 1460 if ((prt_cfg & idlemask) == idlemask) 1461 break; 1462 delay(1); 1463 } 1464 if (timeout == 0) 1465 printf("%s: port idle timeout\n", DEVNAME(sc)); 1466 1467 if (sc->sc_link_duplex) 1468 prt_cfg |= BGX_GMP_GMI_PRT_CFG_DUPLEX; 1469 else 1470 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_DUPLEX; 1471 1472 switch (ifp->if_baudrate) { 1473 case IF_Mbps(10): 1474 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SPEED; 1475 prt_cfg |= BGX_GMP_GMI_PRT_CFG_SPEED_MSB; 1476 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SLOTTIME; 1477 samp_pt = 25; 1478 tx_slot = 0x40; 1479 tx_burst = 0; 1480 break; 1481 case IF_Mbps(100): 1482 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SPEED; 1483 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SPEED_MSB; 1484 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SLOTTIME; 1485 samp_pt = 5; 1486 tx_slot = 0x40; 1487 tx_burst = 0; 1488 break; 1489 case IF_Gbps(1): 1490 default: 1491 prt_cfg |= BGX_GMP_GMI_PRT_CFG_SPEED; 1492 prt_cfg &= ~BGX_GMP_GMI_PRT_CFG_SPEED_MSB; 1493 prt_cfg |= BGX_GMP_GMI_PRT_CFG_SLOTTIME; 1494 samp_pt = 1; 1495 tx_slot = 0x200; 1496 if (sc->sc_link_duplex) 1497 tx_burst = 0; 1498 else 1499 tx_burst = 0x2000; 1500 break; 1501 } 1502 1503 PORT_WR_8(sc, BGX_GMP_GMI_TX_SLOT, tx_slot); 1504 PORT_WR_8(sc, BGX_GMP_GMI_TX_BURST, tx_burst); 1505 1506 misc_ctl = PORT_RD_8(sc, BGX_GMP_PCS_MISC_CTL); 1507 misc_ctl &= ~BGX_GMP_PCS_MISC_CTL_GMXENO; 1508 misc_ctl &= ~BGX_GMP_PCS_MISC_CTL_SAMP_PT_M; 1509 misc_ctl |= samp_pt << BGX_GMP_PCS_MISC_CTL_SAMP_PT_S; 1510 PORT_WR_8(sc, BGX_GMP_PCS_MISC_CTL, misc_ctl); 1511 (void)PORT_RD_8(sc, BGX_GMP_PCS_MISC_CTL); 1512 1513 PORT_WR_8(sc, BGX_GMP_GMI_PRT_CFG, prt_cfg); 1514 (void)PORT_RD_8(sc, BGX_GMP_GMI_PRT_CFG); 1515 1516 config = PORT_RD_8(sc, BGX_CMR_CONFIG); 1517 config |= BGX_CMR_CONFIG_ENABLE | 1518 BGX_CMR_CONFIG_DATA_PKT_RX_EN | 1519 BGX_CMR_CONFIG_DATA_PKT_TX_EN; 1520 PORT_WR_8(sc, BGX_CMR_CONFIG, config); 1521 (void)PORT_RD_8(sc, BGX_CMR_CONFIG); 1522 } 1523 1524 int 1525 ogx_node_init(struct ogx_node **pnode, bus_dma_tag_t dmat, bus_space_tag_t iot) 1526 { 1527 const struct ogx_config *cfg; 1528 struct ogx_node *node = &ogx_node; 1529 uint64_t val; 1530 uint32_t chipid; 1531 int cl, i, timeout; 1532 1533 if (node->node_flags & NODE_INITED) { 1534 *pnode = node; 1535 return 0; 1536 } 1537 1538 chipid = octeon_get_chipid(); 1539 switch (octeon_model_family(chipid)) { 1540 case OCTEON_MODEL_FAMILY_CN73XX: 1541 node->node_cfg = cfg = &ogx_cn73xx_config; 1542 break; 1543 case OCTEON_MODEL_FAMILY_CN78XX: 1544 node->node_cfg = cfg = &ogx_cn78xx_config; 1545 break; 1546 default: 1547 printf(": unhandled chipid 0x%x\n", chipid); 1548 return -1; 1549 } 1550 1551 rw_init(&node->node_lock, "ogxnlk"); 1552 1553 node->node_dmat = dmat; 1554 node->node_iot = iot; 1555 if (bus_space_map(node->node_iot, FPA3_BASE, FPA3_SIZE, 0, 1556 &node->node_fpa3)) { 1557 printf(": can't map FPA3\n"); 1558 goto error; 1559 } 1560 if (bus_space_map(node->node_iot, PKI_BASE, PKI_SIZE, 0, 1561 &node->node_pki)) { 1562 printf(": can't map PKI\n"); 1563 goto error; 1564 } 1565 if (bus_space_map(node->node_iot, PKO3_BASE, PKO3_SIZE, 0, 1566 &node->node_pko3)) { 1567 printf(": can't map PKO3\n"); 1568 goto error; 1569 } 1570 if (bus_space_map(node->node_iot, SSO_BASE, SSO_SIZE, 0, 1571 &node->node_sso)) { 1572 printf(": can't map SSO\n"); 1573 goto error; 1574 } 1575 1576 /* 1577 * The rest of this function handles errors by panicking. 1578 */ 1579 1580 node->node_flags |= NODE_INITED; 1581 1582 PKO3_WR_8(node, PKO3_CHANNEL_LEVEL, 0); 1583 1584 ogx_fpa3_pool_init(node, &node->node_pkt_pool, OGX_POOL_PKT, 1024 * 32); 1585 ogx_fpa3_pool_init(node, &node->node_pko_pool, OGX_POOL_PKO, 1024 * 32); 1586 ogx_fpa3_pool_init(node, &node->node_sso_pool, OGX_POOL_SSO, 1024 * 32); 1587 1588 ogx_fpa3_aura_init(node, &node->node_pko_aura, OGX_AURA_PKO, 1589 &node->node_pko_pool); 1590 ogx_fpa3_aura_init(node, &node->node_sso_aura, OGX_AURA_SSO, 1591 &node->node_sso_pool); 1592 1593 ogx_fpa3_aura_load(node, &node->node_sso_aura, 1024, 4096); 1594 ogx_fpa3_aura_load(node, &node->node_pko_aura, 1024, 4096); 1595 1596 /* 1597 * Initialize the Schedule/Synchronization/Order (SSO) unit. 1598 */ 1599 1600 val = SSO_AW_CFG_LDWB | SSO_AW_CFG_LDT | SSO_AW_CFG_STT; 1601 SSO_WR_8(node, SSO_AW_CFG, val); 1602 1603 val = node->node_id << SSO_XAQ_AURA_NODE_S; 1604 val |= (uint64_t)OGX_AURA_SSO << SSO_XAQ_AURA_LAURA_S; 1605 SSO_WR_8(node, SSO_XAQ_AURA, val); 1606 1607 SSO_WR_8(node, SSO_ERR0, 0); 1608 1609 /* Initialize the hardware's linked lists. */ 1610 for (i = 0; i < 64; i++) { 1611 paddr_t addr; 1612 1613 addr = ogx_fpa3_alloc(&node->node_sso_aura); 1614 if (addr == 0) 1615 panic("%s: could not alloc initial XAQ block %d", 1616 __func__, i); 1617 SSO_WR_8(node, SSO_XAQ_HEAD_PTR(i), addr); 1618 SSO_WR_8(node, SSO_XAQ_TAIL_PTR(i), addr); 1619 SSO_WR_8(node, SSO_XAQ_HEAD_NEXT(i), addr); 1620 SSO_WR_8(node, SSO_XAQ_TAIL_NEXT(i), addr); 1621 1622 SSO_WR_8(node, SSO_GRP_PRI(i), SSO_GRP_PRI_WEIGHT_M); 1623 } 1624 1625 val = SSO_RD_8(node, SSO_AW_CFG); 1626 val |= SSO_AW_CFG_RWEN; 1627 SSO_WR_8(node, SSO_AW_CFG, val); 1628 1629 /* 1630 * Initialize the Packet Input (PKI) unit. 1631 */ 1632 1633 /* Clear any previous style configuration. */ 1634 for (cl = 0; cl < cfg->cfg_nclusters; cl++) { 1635 int pkind; 1636 1637 for (pkind = 0; pkind < 64; pkind++) 1638 PKI_WR_8(node, PKI_CL_PKIND_STYLE(cl, pkind), 0); 1639 } 1640 1641 /* Invalidate all PCAM entries. */ 1642 for (cl = 0; cl < cfg->cfg_nclusters; cl++) { 1643 int bank; 1644 1645 for (bank = 0; bank < 2; bank++) { 1646 for (i = 0; i < 192; i++) { 1647 PKI_WR_8(node, 1648 PKI_CL_PCAM_TERM(cl, bank, i), 0); 1649 } 1650 } 1651 } 1652 1653 PKI_WR_8(node, PKI_STAT_CTL, 0); 1654 1655 /* Enable input backpressure. */ 1656 val = PKI_RD_8(node, PKI_BUF_CTL); 1657 val |= PKI_BUF_CTL_PBP_EN; 1658 PKI_WR_8(node, PKI_BUF_CTL, val); 1659 1660 /* Disable the parsing clusters until the firmware has been loaded. */ 1661 for (cl = 0; cl < cfg->cfg_nclusters; cl++) { 1662 val = PKI_RD_8(node, PKI_ICG_CFG(cl)); 1663 val &= ~PKI_ICG_CFG_PENA; 1664 PKI_WR_8(node, PKI_ICG_CFG(cl), val); 1665 } 1666 1667 val = PKI_RD_8(node, PKI_GBL_PEN); 1668 val &= ~PKI_GBL_PEN_M; 1669 val |= PKI_GBL_PEN_L3; 1670 val |= PKI_GBL_PEN_L4; 1671 PKI_WR_8(node, PKI_GBL_PEN, val); 1672 1673 for (i = 0; i < nitems(ogx_ltypes); i++) { 1674 val = PKI_RD_8(node, PKI_LTYPE_MAP(i)); 1675 val &= ~0x7; 1676 val |= ogx_ltypes[i]; 1677 PKI_WR_8(node, PKI_LTYPE_MAP(i), val); 1678 } 1679 1680 while (PKI_RD_8(node, PKI_SFT_RST) & PKI_SFT_RST_BUSY) 1681 delay(1); 1682 1683 val = PKI_RD_8(node, PKI_BUF_CTL); 1684 val |= PKI_BUF_CTL_PKI_EN; 1685 PKI_WR_8(node, PKI_BUF_CTL, val); 1686 1687 /* 1688 * Initialize the Packet Output (PKO) unit. 1689 */ 1690 1691 /* Detach MACs from FIFOs. */ 1692 for (i = 0; i < cfg->cfg_nmacs; i++) { 1693 val = PKO3_RD_8(node, PKO3_MAC_CFG(i)); 1694 val |= PKO3_MAC_CFG_FIFO_NUM_M; 1695 PKO3_WR_8(node, PKO3_MAC_CFG(i), val); 1696 } 1697 1698 /* Attach port queues to the NULL FIFO. */ 1699 for (i = 0; i < cfg->cfg_npqs; i++) { 1700 val = (uint64_t)cfg->cfg_nullmac << PKO3_L1_SQ_TOPOLOGY_LINK_S; 1701 PKO3_WR_8(node, PKO3_L1_SQ_TOPOLOGY(i), val); 1702 val = (uint64_t)cfg->cfg_nullmac << PKO3_L1_SQ_SHAPE_LINK_S; 1703 PKO3_WR_8(node, PKO3_L1_SQ_SHAPE(i), val); 1704 val = (uint64_t)cfg->cfg_nullmac << PKO3_L1_SQ_LINK_LINK_S; 1705 PKO3_WR_8(node, PKO3_L1_SQ_LINK(i), val); 1706 } 1707 1708 /* Reset the FIFO groups to use 2.5 KB per each FIFO. */ 1709 for (i = 0; i < cfg->cfg_nfifogrps; i++) { 1710 val = PKO3_RD_8(node, PKO3_PTGF_CFG(i)); 1711 val &= ~PKO3_PTGF_CFG_SIZE_M; 1712 val &= ~PKO3_PTGF_CFG_RATE_M; 1713 val |= 2 << PKO3_PTGF_CFG_RATE_S; 1714 val |= PKO3_PTGF_CFG_RESET; 1715 PKO3_WR_8(node, PKO3_PTGF_CFG(i), val); 1716 1717 val = PKO3_RD_8(node, PKO3_PTGF_CFG(i)); 1718 val &= ~PKO3_PTGF_CFG_RESET; 1719 PKO3_WR_8(node, PKO3_PTGF_CFG(i), val); 1720 } 1721 1722 PKO3_WR_8(node, PKO3_DPFI_FLUSH, 0); 1723 1724 /* Set PKO aura. */ 1725 val = ((uint64_t)node->node_id << PKO3_DPFI_FPA_AURA_NODE_S) | 1726 (OGX_AURA_PKO << PKO3_DPFI_FPA_AURA_AURA_S); 1727 PKO3_WR_8(node, PKO3_DPFI_FPA_AURA, val); 1728 1729 /* Allow PKO to use the FPA. */ 1730 PKO3_WR_8(node, PKO3_DPFI_FPA_ENA, PKO3_DPFI_FPA_ENA_ENABLE); 1731 1732 timeout = 1000; 1733 while (timeout-- > 0) { 1734 val = PKO3_RD_8(node, PKO3_STATUS); 1735 if (ISSET(val, PKO3_STATUS_PKO_RDY)) 1736 break; 1737 delay(1000); 1738 } 1739 if (timeout == 0) 1740 panic("PKO timeout"); 1741 1742 val = 72 << PKO3_PTF_IOBP_CFG_MAX_RD_SZ_S; 1743 PKO3_WR_8(node, PKO3_PTF_IOBP_CFG, val); 1744 1745 val = 60 << PKO3_PDM_CFG_MIN_PAD_LEN_S; 1746 PKO3_WR_8(node, PKO3_PDM_CFG, val); 1747 1748 PKO3_WR_8(node, PKO3_ENABLE, PKO3_ENABLE_ENABLE); 1749 1750 *pnode = node; 1751 return 0; 1752 1753 error: 1754 if (node->node_sso != 0) 1755 bus_space_unmap(node->node_iot, node->node_sso, SSO_SIZE); 1756 if (node->node_pko3 != 0) 1757 bus_space_unmap(node->node_iot, node->node_pko3, PKO3_SIZE); 1758 if (node->node_pki != 0) 1759 bus_space_unmap(node->node_iot, node->node_pki, PKI_SIZE); 1760 if (node->node_fpa3 != 0) 1761 bus_space_unmap(node->node_iot, node->node_fpa3, FPA3_SIZE); 1762 node->node_sso = 0; 1763 node->node_pko3 = 0; 1764 node->node_pki = 0; 1765 node->node_fpa3 = 0; 1766 return 1; 1767 } 1768 1769 paddr_t 1770 ogx_fpa3_alloc(struct fpa3aura *aura) 1771 { 1772 uint64_t op; 1773 1774 op = FPA3_LD_IO | FPA3_LD_DID; 1775 op |= (uint64_t)aura->nodeid << FPA3_LD_NODE_S; 1776 op |= (uint64_t)aura->auraid << FPA3_LD_AURA_S; 1777 return octeon_xkphys_read_8(op); 1778 } 1779 1780 void 1781 ogx_fpa3_free(struct fpa3aura *aura, paddr_t addr) 1782 { 1783 uint64_t op; 1784 1785 /* Flush pending writes before the block is freed. */ 1786 octeon_syncw(); 1787 1788 op = FPA3_ST_IO | FPA3_ST_DID_FPA; 1789 op |= (uint64_t)aura->nodeid << FPA3_ST_NODE_S; 1790 op |= (uint64_t)aura->auraid << FPA3_ST_AURA_S; 1791 octeon_xkphys_write_8(op, addr); 1792 } 1793 1794 void 1795 ogx_fpa3_pool_init(struct ogx_node *node, struct fpa3pool *pool, 1796 uint32_t poolid, uint32_t nentries) 1797 { 1798 size_t segsize; 1799 int rsegs; 1800 1801 segsize = nentries * 16; 1802 1803 pool->nodeid = node->node_id; 1804 pool->poolid = poolid; 1805 1806 if (bus_dmamap_create(node->node_dmat, segsize, 1, segsize, 0, 1807 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &pool->dmap)) 1808 panic("%s: out of memory", __func__); 1809 if (bus_dmamem_alloc(node->node_dmat, segsize, CACHELINESIZE, 1810 0, &pool->dmaseg, 1, &rsegs, 1811 BUS_DMA_NOWAIT | BUS_DMA_ZERO)) 1812 panic("%s: out of memory", __func__); 1813 if (bus_dmamem_map(node->node_dmat, &pool->dmaseg, 1, segsize, 1814 &pool->kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) 1815 panic("%s: bus_dmamem_map", __func__); 1816 if (bus_dmamap_load(node->node_dmat, pool->dmap, pool->kva, segsize, 1817 NULL, BUS_DMA_NOWAIT)) 1818 panic("%s: bus_dmamap_load", __func__); 1819 1820 /* Disable the pool before setup. */ 1821 FPA3_WR_8(node, FPA3_POOL_CFG(poolid), 0); 1822 1823 /* Set permitted address range of stored pointers. */ 1824 FPA3_WR_8(node, FPA3_POOL_START_ADDR(poolid), CACHELINESIZE); 1825 FPA3_WR_8(node, FPA3_POOL_END_ADDR(poolid), UINT32_MAX); 1826 1827 /* Set up the pointer stack. */ 1828 FPA3_WR_8(node, FPA3_POOL_STACK_BASE(poolid), pool->dmaseg.ds_addr); 1829 FPA3_WR_8(node, FPA3_POOL_STACK_ADDR(poolid), pool->dmaseg.ds_addr); 1830 FPA3_WR_8(node, FPA3_POOL_STACK_END(poolid), pool->dmaseg.ds_addr + 1831 pool->dmaseg.ds_len); 1832 1833 /* Re-enable the pool. */ 1834 FPA3_WR_8(node, FPA3_POOL_CFG(poolid), FPA3_POOL_CFG_ENA); 1835 } 1836 1837 void 1838 ogx_fpa3_aura_init(struct ogx_node *node, struct fpa3aura *aura, 1839 uint32_t auraid, struct fpa3pool *pool) 1840 { 1841 KASSERT(node->node_id == pool->nodeid); 1842 1843 aura->nodeid = pool->nodeid; 1844 aura->poolid = pool->poolid; 1845 aura->auraid = auraid; 1846 1847 /* Enable pointer counting. */ 1848 FPA3_WR_8(node, FPA3_AURA_CFG(aura->auraid), 0); 1849 FPA3_WR_8(node, FPA3_AURA_CNT(aura->auraid), 1024); 1850 FPA3_WR_8(node, FPA3_AURA_CNT_LIMIT(aura->auraid), 1024); 1851 1852 /* Set the backend pool. */ 1853 FPA3_WR_8(node, FPA3_AURA_POOL(aura->auraid), aura->poolid); 1854 } 1855 1856 void 1857 ogx_fpa3_aura_load(struct ogx_node *node, struct fpa3aura *aura, size_t nelem, 1858 size_t size) 1859 { 1860 paddr_t addr; 1861 caddr_t kva; 1862 size_t i; 1863 size_t totsize; 1864 int rsegs; 1865 1866 KASSERT(size % CACHELINESIZE == 0); 1867 1868 if (nelem > SIZE_MAX / size) 1869 panic("%s: too large allocation", __func__); 1870 totsize = nelem * size; 1871 1872 if (bus_dmamap_create(node->node_dmat, totsize, 1, totsize, 0, 1873 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aura->dmap)) 1874 panic("%s: out of memory", __func__); 1875 if (bus_dmamem_alloc(node->node_dmat, totsize, CACHELINESIZE, 0, 1876 &aura->dmaseg, 1, &rsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO)) 1877 panic("%s: out of memory", __func__); 1878 if (bus_dmamem_map(node->node_dmat, &aura->dmaseg, rsegs, totsize, 1879 &kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) 1880 panic("%s: bus_dmamem_map failed", __func__); 1881 if (bus_dmamap_load(node->node_dmat, aura->dmap, kva, totsize, NULL, 1882 BUS_DMA_NOWAIT)) 1883 panic("%s: bus_dmamap_load failed", __func__); 1884 1885 for (i = 0, addr = aura->dmaseg.ds_addr; i < nelem; i++, addr += size) 1886 ogx_fpa3_free(aura, addr); 1887 } 1888 1889 int 1890 ogx_node_load_firmware(struct ogx_node *node) 1891 { 1892 struct ogx_fwhdr *fw; 1893 uint8_t *ucode = NULL; 1894 size_t size = 0; 1895 uint64_t *imem, val; 1896 int cl, error = 0, i; 1897 1898 rw_enter_write(&node->node_lock); 1899 if (node->node_flags & NODE_FWREADY) 1900 goto out; 1901 1902 error = loadfirmware("ogx-pki-cluster", &ucode, &size); 1903 if (error != 0) { 1904 printf("ogx node%llu: could not load firmware, error %d\n", 1905 node->node_id, error); 1906 goto out; 1907 } 1908 1909 fw = (struct ogx_fwhdr *)ucode; 1910 if (size < sizeof(*fw) || fw->fw_size != size - sizeof(*fw)) { 1911 printf("ogx node%llu: invalid firmware\n", node->node_id); 1912 error = EINVAL; 1913 goto out; 1914 } 1915 1916 imem = (uint64_t *)(fw + 1); 1917 for (i = 0; i < fw->fw_size / sizeof(uint64_t); i++) 1918 PKI_WR_8(node, PKI_IMEM(i), imem[i]); 1919 1920 /* Enable the parsing clusters. */ 1921 for (cl = 0; cl < node->node_cfg->cfg_nclusters; cl++) { 1922 val = PKI_RD_8(node, PKI_ICG_CFG(cl)); 1923 val |= PKI_ICG_CFG_PENA; 1924 PKI_WR_8(node, PKI_ICG_CFG(cl), val); 1925 } 1926 1927 node->node_flags |= NODE_FWREADY; 1928 1929 out: 1930 free(ucode, M_DEVBUF, size); 1931 rw_exit_write(&node->node_lock); 1932 return error; 1933 } 1934