1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Physcial Function ethernet driver 3 * 4 * Copyright (C) 2020 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/interrupt.h> 13 #include <linux/pci.h> 14 #include <linux/etherdevice.h> 15 #include <linux/of.h> 16 #include <linux/if_vlan.h> 17 #include <linux/iommu.h> 18 #include <net/ip.h> 19 20 #include "otx2_reg.h" 21 #include "otx2_common.h" 22 #include "otx2_txrx.h" 23 #include "otx2_struct.h" 24 25 #define DRV_NAME "octeontx2-nicpf" 26 #define DRV_STRING "Marvell OcteonTX2 NIC Physical Function Driver" 27 #define DRV_VERSION "1.0" 28 29 /* Supported devices */ 30 static const struct pci_device_id otx2_pf_id_table[] = { 31 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) }, 32 { 0, } /* end of table */ 33 }; 34 35 MODULE_AUTHOR("Marvell International Ltd."); 36 MODULE_DESCRIPTION(DRV_STRING); 37 MODULE_LICENSE("GPL v2"); 38 MODULE_VERSION(DRV_VERSION); 39 MODULE_DEVICE_TABLE(pci, otx2_pf_id_table); 40 41 enum { 42 TYPE_PFAF, 43 TYPE_PFVF, 44 }; 45 46 static int otx2_change_mtu(struct net_device *netdev, int new_mtu) 47 { 48 bool if_up = netif_running(netdev); 49 int err = 0; 50 51 if (if_up) 52 otx2_stop(netdev); 53 54 netdev_info(netdev, "Changing MTU from %d to %d\n", 55 netdev->mtu, new_mtu); 56 netdev->mtu = new_mtu; 57 58 if (if_up) 59 err = otx2_open(netdev); 60 61 return err; 62 } 63 64 static void otx2_queue_work(struct mbox *mw, struct workqueue_struct *mbox_wq, 65 int first, int mdevs, u64 intr, int type) 66 { 67 struct otx2_mbox_dev *mdev; 68 struct otx2_mbox *mbox; 69 struct mbox_hdr *hdr; 70 int i; 71 72 for (i = first; i < mdevs; i++) { 73 /* start from 0 */ 74 if (!(intr & BIT_ULL(i - first))) 75 continue; 76 77 mbox = &mw->mbox; 78 mdev = &mbox->dev[i]; 79 if (type == TYPE_PFAF) 80 otx2_sync_mbox_bbuf(mbox, i); 81 hdr = mdev->mbase + mbox->rx_start; 82 /* The hdr->num_msgs is set to zero immediately in the interrupt 83 * handler to ensure that it holds a correct value next time 84 * when the interrupt handler is called. 85 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler 86 * pf>mbox.up_num_msgs holds the data for use in 87 * pfaf_mbox_up_handler. 88 */ 89 if (hdr->num_msgs) { 90 mw[i].num_msgs = hdr->num_msgs; 91 hdr->num_msgs = 0; 92 if (type == TYPE_PFAF) 93 memset(mbox->hwbase + mbox->rx_start, 0, 94 ALIGN(sizeof(struct mbox_hdr), 95 sizeof(u64))); 96 97 queue_work(mbox_wq, &mw[i].mbox_wrk); 98 } 99 100 mbox = &mw->mbox_up; 101 mdev = &mbox->dev[i]; 102 if (type == TYPE_PFAF) 103 otx2_sync_mbox_bbuf(mbox, i); 104 hdr = mdev->mbase + mbox->rx_start; 105 if (hdr->num_msgs) { 106 mw[i].up_num_msgs = hdr->num_msgs; 107 hdr->num_msgs = 0; 108 if (type == TYPE_PFAF) 109 memset(mbox->hwbase + mbox->rx_start, 0, 110 ALIGN(sizeof(struct mbox_hdr), 111 sizeof(u64))); 112 113 queue_work(mbox_wq, &mw[i].mbox_up_wrk); 114 } 115 } 116 } 117 118 static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf, 119 struct mbox_msghdr *msg) 120 { 121 if (msg->id >= MBOX_MSG_MAX) { 122 dev_err(pf->dev, 123 "Mbox msg with unknown ID 0x%x\n", msg->id); 124 return; 125 } 126 127 if (msg->sig != OTX2_MBOX_RSP_SIG) { 128 dev_err(pf->dev, 129 "Mbox msg with wrong signature %x, ID 0x%x\n", 130 msg->sig, msg->id); 131 return; 132 } 133 134 switch (msg->id) { 135 case MBOX_MSG_READY: 136 pf->pcifunc = msg->pcifunc; 137 break; 138 case MBOX_MSG_MSIX_OFFSET: 139 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg); 140 break; 141 case MBOX_MSG_NPA_LF_ALLOC: 142 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg); 143 break; 144 case MBOX_MSG_NIX_LF_ALLOC: 145 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg); 146 break; 147 case MBOX_MSG_NIX_TXSCH_ALLOC: 148 mbox_handler_nix_txsch_alloc(pf, 149 (struct nix_txsch_alloc_rsp *)msg); 150 break; 151 case MBOX_MSG_CGX_STATS: 152 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg); 153 break; 154 default: 155 if (msg->rc) 156 dev_err(pf->dev, 157 "Mbox msg response has err %d, ID 0x%x\n", 158 msg->rc, msg->id); 159 break; 160 } 161 } 162 163 static void otx2_pfaf_mbox_handler(struct work_struct *work) 164 { 165 struct otx2_mbox_dev *mdev; 166 struct mbox_hdr *rsp_hdr; 167 struct mbox_msghdr *msg; 168 struct otx2_mbox *mbox; 169 struct mbox *af_mbox; 170 struct otx2_nic *pf; 171 int offset, id; 172 173 af_mbox = container_of(work, struct mbox, mbox_wrk); 174 mbox = &af_mbox->mbox; 175 mdev = &mbox->dev[0]; 176 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); 177 178 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); 179 pf = af_mbox->pfvf; 180 181 for (id = 0; id < af_mbox->num_msgs; id++) { 182 msg = (struct mbox_msghdr *)(mdev->mbase + offset); 183 otx2_process_pfaf_mbox_msg(pf, msg); 184 offset = mbox->rx_start + msg->next_msgoff; 185 mdev->msgs_acked++; 186 } 187 188 otx2_mbox_reset(mbox, 0); 189 } 190 191 static void otx2_handle_link_event(struct otx2_nic *pf) 192 { 193 struct cgx_link_user_info *linfo = &pf->linfo; 194 struct net_device *netdev = pf->netdev; 195 196 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name, 197 linfo->link_up ? "UP" : "DOWN", linfo->speed, 198 linfo->full_duplex ? "Full" : "Half"); 199 if (linfo->link_up) { 200 netif_carrier_on(netdev); 201 netif_tx_start_all_queues(netdev); 202 } else { 203 netif_tx_stop_all_queues(netdev); 204 netif_carrier_off(netdev); 205 } 206 } 207 208 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf, 209 struct cgx_link_info_msg *msg, 210 struct msg_rsp *rsp) 211 { 212 /* Copy the link info sent by AF */ 213 pf->linfo = msg->link_info; 214 215 /* interface has not been fully configured yet */ 216 if (pf->flags & OTX2_FLAG_INTF_DOWN) 217 return 0; 218 219 otx2_handle_link_event(pf); 220 return 0; 221 } 222 223 static int otx2_process_mbox_msg_up(struct otx2_nic *pf, 224 struct mbox_msghdr *req) 225 { 226 /* Check if valid, if not reply with a invalid msg */ 227 if (req->sig != OTX2_MBOX_REQ_SIG) { 228 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id); 229 return -ENODEV; 230 } 231 232 switch (req->id) { 233 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ 234 case _id: { \ 235 struct _rsp_type *rsp; \ 236 int err; \ 237 \ 238 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \ 239 &pf->mbox.mbox_up, 0, \ 240 sizeof(struct _rsp_type)); \ 241 if (!rsp) \ 242 return -ENOMEM; \ 243 \ 244 rsp->hdr.id = _id; \ 245 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \ 246 rsp->hdr.pcifunc = 0; \ 247 rsp->hdr.rc = 0; \ 248 \ 249 err = otx2_mbox_up_handler_ ## _fn_name( \ 250 pf, (struct _req_type *)req, rsp); \ 251 return err; \ 252 } 253 MBOX_UP_CGX_MESSAGES 254 #undef M 255 break; 256 default: 257 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id); 258 return -ENODEV; 259 } 260 return 0; 261 } 262 263 static void otx2_pfaf_mbox_up_handler(struct work_struct *work) 264 { 265 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk); 266 struct otx2_mbox *mbox = &af_mbox->mbox_up; 267 struct otx2_mbox_dev *mdev = &mbox->dev[0]; 268 struct otx2_nic *pf = af_mbox->pfvf; 269 int offset, id, devid = 0; 270 struct mbox_hdr *rsp_hdr; 271 struct mbox_msghdr *msg; 272 273 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); 274 275 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); 276 277 for (id = 0; id < af_mbox->up_num_msgs; id++) { 278 msg = (struct mbox_msghdr *)(mdev->mbase + offset); 279 280 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK; 281 /* Skip processing VF's messages */ 282 if (!devid) 283 otx2_process_mbox_msg_up(pf, msg); 284 offset = mbox->rx_start + msg->next_msgoff; 285 } 286 287 otx2_mbox_msg_send(mbox, 0); 288 } 289 290 static irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq) 291 { 292 struct otx2_nic *pf = (struct otx2_nic *)pf_irq; 293 struct mbox *mbox; 294 295 /* Clear the IRQ */ 296 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0)); 297 298 mbox = &pf->mbox; 299 otx2_queue_work(mbox, pf->mbox_wq, 0, 1, 1, TYPE_PFAF); 300 301 return IRQ_HANDLED; 302 } 303 304 static void otx2_disable_mbox_intr(struct otx2_nic *pf) 305 { 306 int vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX); 307 308 /* Disable AF => PF mailbox IRQ */ 309 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0)); 310 free_irq(vector, pf); 311 } 312 313 static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af) 314 { 315 struct otx2_hw *hw = &pf->hw; 316 struct msg_req *req; 317 char *irq_name; 318 int err; 319 320 /* Register mailbox interrupt handler */ 321 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE]; 322 snprintf(irq_name, NAME_SIZE, "RVUPFAF Mbox"); 323 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX), 324 otx2_pfaf_mbox_intr_handler, 0, irq_name, pf); 325 if (err) { 326 dev_err(pf->dev, 327 "RVUPF: IRQ registration failed for PFAF mbox irq\n"); 328 return err; 329 } 330 331 /* Enable mailbox interrupt for msgs coming from AF. 332 * First clear to avoid spurious interrupts, if any. 333 */ 334 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0)); 335 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0)); 336 337 if (!probe_af) 338 return 0; 339 340 /* Check mailbox communication with AF */ 341 req = otx2_mbox_alloc_msg_ready(&pf->mbox); 342 if (!req) { 343 otx2_disable_mbox_intr(pf); 344 return -ENOMEM; 345 } 346 err = otx2_sync_mbox_msg(&pf->mbox); 347 if (err) { 348 dev_warn(pf->dev, 349 "AF not responding to mailbox, deferring probe\n"); 350 otx2_disable_mbox_intr(pf); 351 return -EPROBE_DEFER; 352 } 353 354 return 0; 355 } 356 357 static void otx2_pfaf_mbox_destroy(struct otx2_nic *pf) 358 { 359 struct mbox *mbox = &pf->mbox; 360 361 if (pf->mbox_wq) { 362 flush_workqueue(pf->mbox_wq); 363 destroy_workqueue(pf->mbox_wq); 364 pf->mbox_wq = NULL; 365 } 366 367 if (mbox->mbox.hwbase) 368 iounmap((void __iomem *)mbox->mbox.hwbase); 369 370 otx2_mbox_destroy(&mbox->mbox); 371 otx2_mbox_destroy(&mbox->mbox_up); 372 } 373 374 static int otx2_pfaf_mbox_init(struct otx2_nic *pf) 375 { 376 struct mbox *mbox = &pf->mbox; 377 void __iomem *hwbase; 378 int err; 379 380 mbox->pfvf = pf; 381 pf->mbox_wq = alloc_workqueue("otx2_pfaf_mailbox", 382 WQ_UNBOUND | WQ_HIGHPRI | 383 WQ_MEM_RECLAIM, 1); 384 if (!pf->mbox_wq) 385 return -ENOMEM; 386 387 /* Mailbox is a reserved memory (in RAM) region shared between 388 * admin function (i.e AF) and this PF, shouldn't be mapped as 389 * device memory to allow unaligned accesses. 390 */ 391 hwbase = ioremap_wc(pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM), 392 pci_resource_len(pf->pdev, PCI_MBOX_BAR_NUM)); 393 if (!hwbase) { 394 dev_err(pf->dev, "Unable to map PFAF mailbox region\n"); 395 err = -ENOMEM; 396 goto exit; 397 } 398 399 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base, 400 MBOX_DIR_PFAF, 1); 401 if (err) 402 goto exit; 403 404 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base, 405 MBOX_DIR_PFAF_UP, 1); 406 if (err) 407 goto exit; 408 409 err = otx2_mbox_bbuf_init(mbox, pf->pdev); 410 if (err) 411 goto exit; 412 413 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler); 414 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler); 415 otx2_mbox_lock_init(&pf->mbox); 416 417 return 0; 418 exit: 419 otx2_pfaf_mbox_destroy(pf); 420 return err; 421 } 422 423 static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable) 424 { 425 struct msg_req *msg; 426 int err; 427 428 otx2_mbox_lock(&pf->mbox); 429 if (enable) 430 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox); 431 else 432 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox); 433 434 if (!msg) { 435 otx2_mbox_unlock(&pf->mbox); 436 return -ENOMEM; 437 } 438 439 err = otx2_sync_mbox_msg(&pf->mbox); 440 otx2_mbox_unlock(&pf->mbox); 441 return err; 442 } 443 444 static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable) 445 { 446 struct msg_req *msg; 447 int err; 448 449 otx2_mbox_lock(&pf->mbox); 450 if (enable) 451 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox); 452 else 453 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox); 454 455 if (!msg) { 456 otx2_mbox_unlock(&pf->mbox); 457 return -ENOMEM; 458 } 459 460 err = otx2_sync_mbox_msg(&pf->mbox); 461 otx2_mbox_unlock(&pf->mbox); 462 return err; 463 } 464 465 int otx2_set_real_num_queues(struct net_device *netdev, 466 int tx_queues, int rx_queues) 467 { 468 int err; 469 470 err = netif_set_real_num_tx_queues(netdev, tx_queues); 471 if (err) { 472 netdev_err(netdev, 473 "Failed to set no of Tx queues: %d\n", tx_queues); 474 return err; 475 } 476 477 err = netif_set_real_num_rx_queues(netdev, rx_queues); 478 if (err) 479 netdev_err(netdev, 480 "Failed to set no of Rx queues: %d\n", rx_queues); 481 return err; 482 } 483 484 static irqreturn_t otx2_q_intr_handler(int irq, void *data) 485 { 486 struct otx2_nic *pf = data; 487 u64 val, *ptr; 488 u64 qidx = 0; 489 490 /* CQ */ 491 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) { 492 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT); 493 val = otx2_atomic64_add((qidx << 44), ptr); 494 495 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) | 496 (val & NIX_CQERRINT_BITS)); 497 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42)))) 498 continue; 499 500 if (val & BIT_ULL(42)) { 501 netdev_err(pf->netdev, "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n", 502 qidx, otx2_read64(pf, NIX_LF_ERR_INT)); 503 } else { 504 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR)) 505 netdev_err(pf->netdev, "CQ%lld: Doorbell error", 506 qidx); 507 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT)) 508 netdev_err(pf->netdev, "CQ%lld: Memory fault on CQE write to LLC/DRAM", 509 qidx); 510 } 511 512 schedule_work(&pf->reset_task); 513 } 514 515 /* SQ */ 516 for (qidx = 0; qidx < pf->hw.tx_queues; qidx++) { 517 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT); 518 val = otx2_atomic64_add((qidx << 44), ptr); 519 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) | 520 (val & NIX_SQINT_BITS)); 521 522 if (!(val & (NIX_SQINT_BITS | BIT_ULL(42)))) 523 continue; 524 525 if (val & BIT_ULL(42)) { 526 netdev_err(pf->netdev, "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n", 527 qidx, otx2_read64(pf, NIX_LF_ERR_INT)); 528 } else { 529 if (val & BIT_ULL(NIX_SQINT_LMT_ERR)) { 530 netdev_err(pf->netdev, "SQ%lld: LMT store error NIX_LF_SQ_OP_ERR_DBG:0x%llx", 531 qidx, 532 otx2_read64(pf, 533 NIX_LF_SQ_OP_ERR_DBG)); 534 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, 535 BIT_ULL(44)); 536 } 537 if (val & BIT_ULL(NIX_SQINT_MNQ_ERR)) { 538 netdev_err(pf->netdev, "SQ%lld: Meta-descriptor enqueue error NIX_LF_MNQ_ERR_DGB:0x%llx\n", 539 qidx, 540 otx2_read64(pf, NIX_LF_MNQ_ERR_DBG)); 541 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, 542 BIT_ULL(44)); 543 } 544 if (val & BIT_ULL(NIX_SQINT_SEND_ERR)) { 545 netdev_err(pf->netdev, "SQ%lld: Send error, NIX_LF_SEND_ERR_DBG 0x%llx", 546 qidx, 547 otx2_read64(pf, 548 NIX_LF_SEND_ERR_DBG)); 549 otx2_write64(pf, NIX_LF_SEND_ERR_DBG, 550 BIT_ULL(44)); 551 } 552 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) 553 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", 554 qidx); 555 } 556 557 schedule_work(&pf->reset_task); 558 } 559 560 return IRQ_HANDLED; 561 } 562 563 static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq) 564 { 565 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq; 566 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev; 567 int qidx = cq_poll->cint_idx; 568 569 /* Disable interrupts. 570 * 571 * Completion interrupts behave in a level-triggered interrupt 572 * fashion, and hence have to be cleared only after it is serviced. 573 */ 574 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); 575 576 /* Schedule NAPI */ 577 napi_schedule_irqoff(&cq_poll->napi); 578 579 return IRQ_HANDLED; 580 } 581 582 static void otx2_disable_napi(struct otx2_nic *pf) 583 { 584 struct otx2_qset *qset = &pf->qset; 585 struct otx2_cq_poll *cq_poll; 586 int qidx; 587 588 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { 589 cq_poll = &qset->napi[qidx]; 590 napi_disable(&cq_poll->napi); 591 netif_napi_del(&cq_poll->napi); 592 } 593 } 594 595 static void otx2_free_cq_res(struct otx2_nic *pf) 596 { 597 struct otx2_qset *qset = &pf->qset; 598 struct otx2_cq_queue *cq; 599 int qidx; 600 601 /* Disable CQs */ 602 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false); 603 for (qidx = 0; qidx < qset->cq_cnt; qidx++) { 604 cq = &qset->cq[qidx]; 605 qmem_free(pf->dev, cq->cqe); 606 } 607 } 608 609 static void otx2_free_sq_res(struct otx2_nic *pf) 610 { 611 struct otx2_qset *qset = &pf->qset; 612 struct otx2_snd_queue *sq; 613 int qidx; 614 615 /* Disable SQs */ 616 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false); 617 /* Free SQB pointers */ 618 otx2_sq_free_sqbs(pf); 619 for (qidx = 0; qidx < pf->hw.tx_queues; qidx++) { 620 sq = &qset->sq[qidx]; 621 qmem_free(pf->dev, sq->sqe); 622 qmem_free(pf->dev, sq->tso_hdrs); 623 kfree(sq->sg); 624 kfree(sq->sqb_ptrs); 625 } 626 } 627 628 static int otx2_init_hw_resources(struct otx2_nic *pf) 629 { 630 struct mbox *mbox = &pf->mbox; 631 struct otx2_hw *hw = &pf->hw; 632 struct msg_req *req; 633 int err = 0, lvl; 634 635 /* Set required NPA LF's pool counts 636 * Auras and Pools are used in a 1:1 mapping, 637 * so, aura count = pool count. 638 */ 639 hw->rqpool_cnt = hw->rx_queues; 640 hw->sqpool_cnt = hw->tx_queues; 641 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt; 642 643 /* Get the size of receive buffers to allocate */ 644 pf->rbsize = RCV_FRAG_LEN(pf->netdev->mtu + OTX2_ETH_HLEN); 645 646 otx2_mbox_lock(mbox); 647 /* NPA init */ 648 err = otx2_config_npa(pf); 649 if (err) 650 goto exit; 651 652 /* NIX init */ 653 err = otx2_config_nix(pf); 654 if (err) 655 goto err_free_npa_lf; 656 657 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */ 658 err = otx2_rq_aura_pool_init(pf); 659 if (err) { 660 otx2_mbox_unlock(mbox); 661 goto err_free_nix_lf; 662 } 663 /* Init Auras and pools used by NIX SQ, for queueing SQEs */ 664 err = otx2_sq_aura_pool_init(pf); 665 if (err) { 666 otx2_mbox_unlock(mbox); 667 goto err_free_rq_ptrs; 668 } 669 670 err = otx2_txsch_alloc(pf); 671 if (err) { 672 otx2_mbox_unlock(mbox); 673 goto err_free_sq_ptrs; 674 } 675 676 err = otx2_config_nix_queues(pf); 677 if (err) { 678 otx2_mbox_unlock(mbox); 679 goto err_free_txsch; 680 } 681 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 682 err = otx2_txschq_config(pf, lvl); 683 if (err) { 684 otx2_mbox_unlock(mbox); 685 goto err_free_nix_queues; 686 } 687 } 688 otx2_mbox_unlock(mbox); 689 return err; 690 691 err_free_nix_queues: 692 otx2_free_sq_res(pf); 693 otx2_free_cq_res(pf); 694 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false); 695 err_free_txsch: 696 if (otx2_txschq_stop(pf)) 697 dev_err(pf->dev, "%s failed to stop TX schedulers\n", __func__); 698 err_free_sq_ptrs: 699 otx2_sq_free_sqbs(pf); 700 err_free_rq_ptrs: 701 otx2_free_aura_ptr(pf, AURA_NIX_RQ); 702 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true); 703 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true); 704 otx2_aura_pool_free(pf); 705 err_free_nix_lf: 706 otx2_mbox_lock(mbox); 707 req = otx2_mbox_alloc_msg_nix_lf_free(mbox); 708 if (req) { 709 if (otx2_sync_mbox_msg(mbox)) 710 dev_err(pf->dev, "%s failed to free nixlf\n", __func__); 711 } 712 err_free_npa_lf: 713 /* Reset NPA LF */ 714 req = otx2_mbox_alloc_msg_npa_lf_free(mbox); 715 if (req) { 716 if (otx2_sync_mbox_msg(mbox)) 717 dev_err(pf->dev, "%s failed to free npalf\n", __func__); 718 } 719 exit: 720 otx2_mbox_unlock(mbox); 721 return err; 722 } 723 724 static void otx2_free_hw_resources(struct otx2_nic *pf) 725 { 726 struct otx2_qset *qset = &pf->qset; 727 struct mbox *mbox = &pf->mbox; 728 struct otx2_cq_queue *cq; 729 struct msg_req *req; 730 int qidx, err; 731 732 /* Ensure all SQE are processed */ 733 otx2_sqb_flush(pf); 734 735 /* Stop transmission */ 736 err = otx2_txschq_stop(pf); 737 if (err) 738 dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n"); 739 740 /* Disable RQs */ 741 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false); 742 743 /*Dequeue all CQEs */ 744 for (qidx = 0; qidx < qset->cq_cnt; qidx++) { 745 cq = &qset->cq[qidx]; 746 if (cq->cq_type == CQ_RX) 747 otx2_cleanup_rx_cqes(pf, cq); 748 else 749 otx2_cleanup_tx_cqes(pf, cq); 750 } 751 752 otx2_free_sq_res(pf); 753 754 /* Free RQ buffer pointers*/ 755 otx2_free_aura_ptr(pf, AURA_NIX_RQ); 756 757 otx2_free_cq_res(pf); 758 759 otx2_mbox_lock(mbox); 760 /* Reset NIX LF */ 761 req = otx2_mbox_alloc_msg_nix_lf_free(mbox); 762 if (req) { 763 if (otx2_sync_mbox_msg(mbox)) 764 dev_err(pf->dev, "%s failed to free nixlf\n", __func__); 765 } 766 otx2_mbox_unlock(mbox); 767 768 /* Disable NPA Pool and Aura hw context */ 769 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true); 770 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true); 771 otx2_aura_pool_free(pf); 772 773 otx2_mbox_lock(mbox); 774 /* Reset NPA LF */ 775 req = otx2_mbox_alloc_msg_npa_lf_free(mbox); 776 if (req) { 777 if (otx2_sync_mbox_msg(mbox)) 778 dev_err(pf->dev, "%s failed to free npalf\n", __func__); 779 } 780 otx2_mbox_unlock(mbox); 781 } 782 783 int otx2_open(struct net_device *netdev) 784 { 785 struct otx2_nic *pf = netdev_priv(netdev); 786 struct otx2_cq_poll *cq_poll = NULL; 787 struct otx2_qset *qset = &pf->qset; 788 int err = 0, qidx, vec; 789 char *irq_name; 790 791 netif_carrier_off(netdev); 792 793 pf->qset.cq_cnt = pf->hw.rx_queues + pf->hw.tx_queues; 794 /* RQ and SQs are mapped to different CQs, 795 * so find out max CQ IRQs (i.e CINTs) needed. 796 */ 797 pf->hw.cint_cnt = max(pf->hw.rx_queues, pf->hw.tx_queues); 798 qset->napi = kcalloc(pf->hw.cint_cnt, sizeof(*cq_poll), GFP_KERNEL); 799 if (!qset->napi) 800 return -ENOMEM; 801 802 /* CQ size of RQ */ 803 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256); 804 /* CQ size of SQ */ 805 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K); 806 807 err = -ENOMEM; 808 qset->cq = kcalloc(pf->qset.cq_cnt, 809 sizeof(struct otx2_cq_queue), GFP_KERNEL); 810 if (!qset->cq) 811 goto err_free_mem; 812 813 qset->sq = kcalloc(pf->hw.tx_queues, 814 sizeof(struct otx2_snd_queue), GFP_KERNEL); 815 if (!qset->sq) 816 goto err_free_mem; 817 818 qset->rq = kcalloc(pf->hw.rx_queues, 819 sizeof(struct otx2_rcv_queue), GFP_KERNEL); 820 if (!qset->rq) 821 goto err_free_mem; 822 823 err = otx2_init_hw_resources(pf); 824 if (err) 825 goto err_free_mem; 826 827 /* Register NAPI handler */ 828 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { 829 cq_poll = &qset->napi[qidx]; 830 cq_poll->cint_idx = qidx; 831 /* RQ0 & SQ0 are mapped to CINT0 and so on.. 832 * 'cq_ids[0]' points to RQ's CQ and 833 * 'cq_ids[1]' points to SQ's CQ and 834 */ 835 cq_poll->cq_ids[CQ_RX] = 836 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ; 837 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ? 838 qidx + pf->hw.rx_queues : CINT_INVALID_CQ; 839 cq_poll->dev = (void *)pf; 840 netif_napi_add(netdev, &cq_poll->napi, 841 otx2_napi_handler, NAPI_POLL_WEIGHT); 842 napi_enable(&cq_poll->napi); 843 } 844 845 /* Set maximum frame size allowed in HW */ 846 err = otx2_hw_set_mtu(pf, netdev->mtu); 847 if (err) 848 goto err_disable_napi; 849 850 /* Initialize RSS */ 851 err = otx2_rss_init(pf); 852 if (err) 853 goto err_disable_napi; 854 855 /* Register Queue IRQ handlers */ 856 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START; 857 irq_name = &pf->hw.irq_name[vec * NAME_SIZE]; 858 859 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name); 860 861 err = request_irq(pci_irq_vector(pf->pdev, vec), 862 otx2_q_intr_handler, 0, irq_name, pf); 863 if (err) { 864 dev_err(pf->dev, 865 "RVUPF%d: IRQ registration failed for QERR\n", 866 rvu_get_pf(pf->pcifunc)); 867 goto err_disable_napi; 868 } 869 870 /* Enable QINT IRQ */ 871 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0)); 872 873 /* Register CQ IRQ handlers */ 874 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START; 875 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { 876 irq_name = &pf->hw.irq_name[vec * NAME_SIZE]; 877 878 snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d", pf->netdev->name, 879 qidx); 880 881 err = request_irq(pci_irq_vector(pf->pdev, vec), 882 otx2_cq_intr_handler, 0, irq_name, 883 &qset->napi[qidx]); 884 if (err) { 885 dev_err(pf->dev, 886 "RVUPF%d: IRQ registration failed for CQ%d\n", 887 rvu_get_pf(pf->pcifunc), qidx); 888 goto err_free_cints; 889 } 890 vec++; 891 892 otx2_config_irq_coalescing(pf, qidx); 893 894 /* Enable CQ IRQ */ 895 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0)); 896 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0)); 897 } 898 899 otx2_set_cints_affinity(pf); 900 901 pf->flags &= ~OTX2_FLAG_INTF_DOWN; 902 /* 'intf_down' may be checked on any cpu */ 903 smp_wmb(); 904 905 /* we have already received link status notification */ 906 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK)) 907 otx2_handle_link_event(pf); 908 909 err = otx2_rxtx_enable(pf, true); 910 if (err) 911 goto err_free_cints; 912 913 return 0; 914 915 err_free_cints: 916 otx2_free_cints(pf, qidx); 917 vec = pci_irq_vector(pf->pdev, 918 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); 919 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); 920 synchronize_irq(vec); 921 free_irq(vec, pf); 922 err_disable_napi: 923 otx2_disable_napi(pf); 924 otx2_free_hw_resources(pf); 925 err_free_mem: 926 kfree(qset->sq); 927 kfree(qset->cq); 928 kfree(qset->rq); 929 kfree(qset->napi); 930 return err; 931 } 932 933 int otx2_stop(struct net_device *netdev) 934 { 935 struct otx2_nic *pf = netdev_priv(netdev); 936 struct otx2_cq_poll *cq_poll = NULL; 937 struct otx2_qset *qset = &pf->qset; 938 int qidx, vec, wrk; 939 940 netif_carrier_off(netdev); 941 netif_tx_stop_all_queues(netdev); 942 943 pf->flags |= OTX2_FLAG_INTF_DOWN; 944 /* 'intf_down' may be checked on any cpu */ 945 smp_wmb(); 946 947 /* First stop packet Rx/Tx */ 948 otx2_rxtx_enable(pf, false); 949 950 /* Cleanup Queue IRQ */ 951 vec = pci_irq_vector(pf->pdev, 952 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); 953 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); 954 synchronize_irq(vec); 955 free_irq(vec, pf); 956 957 /* Cleanup CQ NAPI and IRQ */ 958 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START; 959 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { 960 /* Disable interrupt */ 961 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); 962 963 synchronize_irq(pci_irq_vector(pf->pdev, vec)); 964 965 cq_poll = &qset->napi[qidx]; 966 napi_synchronize(&cq_poll->napi); 967 vec++; 968 } 969 970 netif_tx_disable(netdev); 971 972 otx2_free_hw_resources(pf); 973 otx2_free_cints(pf, pf->hw.cint_cnt); 974 otx2_disable_napi(pf); 975 976 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++) 977 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx)); 978 979 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++) 980 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work); 981 devm_kfree(pf->dev, pf->refill_wrk); 982 983 kfree(qset->sq); 984 kfree(qset->cq); 985 kfree(qset->rq); 986 kfree(qset->napi); 987 /* Do not clear RQ/SQ ringsize settings */ 988 memset((void *)qset + offsetof(struct otx2_qset, sqe_cnt), 0, 989 sizeof(*qset) - offsetof(struct otx2_qset, sqe_cnt)); 990 return 0; 991 } 992 993 static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev) 994 { 995 struct otx2_nic *pf = netdev_priv(netdev); 996 int qidx = skb_get_queue_mapping(skb); 997 struct otx2_snd_queue *sq; 998 struct netdev_queue *txq; 999 1000 /* Check for minimum and maximum packet length */ 1001 if (skb->len <= ETH_HLEN || 1002 (!skb_shinfo(skb)->gso_size && skb->len > pf->max_frs)) { 1003 dev_kfree_skb(skb); 1004 return NETDEV_TX_OK; 1005 } 1006 1007 sq = &pf->qset.sq[qidx]; 1008 txq = netdev_get_tx_queue(netdev, qidx); 1009 1010 if (!otx2_sq_append_skb(netdev, sq, skb, qidx)) { 1011 netif_tx_stop_queue(txq); 1012 1013 /* Check again, incase SQBs got freed up */ 1014 smp_mb(); 1015 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb) 1016 > sq->sqe_thresh) 1017 netif_tx_wake_queue(txq); 1018 1019 return NETDEV_TX_BUSY; 1020 } 1021 1022 return NETDEV_TX_OK; 1023 } 1024 1025 static void otx2_set_rx_mode(struct net_device *netdev) 1026 { 1027 struct otx2_nic *pf = netdev_priv(netdev); 1028 struct nix_rx_mode *req; 1029 1030 if (!(netdev->flags & IFF_UP)) 1031 return; 1032 1033 otx2_mbox_lock(&pf->mbox); 1034 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox); 1035 if (!req) { 1036 otx2_mbox_unlock(&pf->mbox); 1037 return; 1038 } 1039 1040 req->mode = NIX_RX_MODE_UCAST; 1041 1042 /* We don't support MAC address filtering yet */ 1043 if (netdev->flags & IFF_PROMISC) 1044 req->mode |= NIX_RX_MODE_PROMISC; 1045 else if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST)) 1046 req->mode |= NIX_RX_MODE_ALLMULTI; 1047 1048 otx2_sync_mbox_msg(&pf->mbox); 1049 otx2_mbox_unlock(&pf->mbox); 1050 } 1051 1052 static int otx2_set_features(struct net_device *netdev, 1053 netdev_features_t features) 1054 { 1055 netdev_features_t changed = features ^ netdev->features; 1056 struct otx2_nic *pf = netdev_priv(netdev); 1057 1058 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev)) 1059 return otx2_cgx_config_loopback(pf, 1060 features & NETIF_F_LOOPBACK); 1061 return 0; 1062 } 1063 1064 static void otx2_reset_task(struct work_struct *work) 1065 { 1066 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task); 1067 1068 if (!netif_running(pf->netdev)) 1069 return; 1070 1071 otx2_stop(pf->netdev); 1072 pf->reset_count++; 1073 otx2_open(pf->netdev); 1074 netif_trans_update(pf->netdev); 1075 } 1076 1077 static const struct net_device_ops otx2_netdev_ops = { 1078 .ndo_open = otx2_open, 1079 .ndo_stop = otx2_stop, 1080 .ndo_start_xmit = otx2_xmit, 1081 .ndo_set_mac_address = otx2_set_mac_address, 1082 .ndo_change_mtu = otx2_change_mtu, 1083 .ndo_set_rx_mode = otx2_set_rx_mode, 1084 .ndo_set_features = otx2_set_features, 1085 .ndo_tx_timeout = otx2_tx_timeout, 1086 .ndo_get_stats64 = otx2_get_stats64, 1087 }; 1088 1089 static int otx2_check_pf_usable(struct otx2_nic *nic) 1090 { 1091 u64 rev; 1092 1093 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM)); 1094 rev = (rev >> 12) & 0xFF; 1095 /* Check if AF has setup revision for RVUM block, 1096 * otherwise this driver probe should be deferred 1097 * until AF driver comes up. 1098 */ 1099 if (!rev) { 1100 dev_warn(nic->dev, 1101 "AF is not initialized, deferring probe\n"); 1102 return -EPROBE_DEFER; 1103 } 1104 return 0; 1105 } 1106 1107 static int otx2_realloc_msix_vectors(struct otx2_nic *pf) 1108 { 1109 struct otx2_hw *hw = &pf->hw; 1110 int num_vec, err; 1111 1112 /* NPA interrupts are inot registered, so alloc only 1113 * upto NIX vector offset. 1114 */ 1115 num_vec = hw->nix_msixoff; 1116 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues; 1117 1118 otx2_disable_mbox_intr(pf); 1119 pci_free_irq_vectors(hw->pdev); 1120 pci_free_irq_vectors(hw->pdev); 1121 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX); 1122 if (err < 0) { 1123 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n", 1124 __func__, num_vec); 1125 return err; 1126 } 1127 1128 return otx2_register_mbox_intr(pf, false); 1129 } 1130 1131 static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1132 { 1133 struct device *dev = &pdev->dev; 1134 struct net_device *netdev; 1135 struct otx2_nic *pf; 1136 struct otx2_hw *hw; 1137 int err, qcount; 1138 int num_vec; 1139 1140 err = pcim_enable_device(pdev); 1141 if (err) { 1142 dev_err(dev, "Failed to enable PCI device\n"); 1143 return err; 1144 } 1145 1146 err = pci_request_regions(pdev, DRV_NAME); 1147 if (err) { 1148 dev_err(dev, "PCI request regions failed 0x%x\n", err); 1149 return err; 1150 } 1151 1152 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); 1153 if (err) { 1154 dev_err(dev, "DMA mask config failed, abort\n"); 1155 goto err_release_regions; 1156 } 1157 1158 pci_set_master(pdev); 1159 1160 /* Set number of queues */ 1161 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT); 1162 1163 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount, qcount); 1164 if (!netdev) { 1165 err = -ENOMEM; 1166 goto err_release_regions; 1167 } 1168 1169 pci_set_drvdata(pdev, netdev); 1170 SET_NETDEV_DEV(netdev, &pdev->dev); 1171 pf = netdev_priv(netdev); 1172 pf->netdev = netdev; 1173 pf->pdev = pdev; 1174 pf->dev = dev; 1175 pf->flags |= OTX2_FLAG_INTF_DOWN; 1176 1177 hw = &pf->hw; 1178 hw->pdev = pdev; 1179 hw->rx_queues = qcount; 1180 hw->tx_queues = qcount; 1181 hw->max_queues = qcount; 1182 1183 num_vec = pci_msix_vec_count(pdev); 1184 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE, 1185 GFP_KERNEL); 1186 if (!hw->irq_name) 1187 goto err_free_netdev; 1188 1189 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec, 1190 sizeof(cpumask_var_t), GFP_KERNEL); 1191 if (!hw->affinity_mask) 1192 goto err_free_netdev; 1193 1194 /* Map CSRs */ 1195 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); 1196 if (!pf->reg_base) { 1197 dev_err(dev, "Unable to map physical function CSRs, aborting\n"); 1198 err = -ENOMEM; 1199 goto err_free_netdev; 1200 } 1201 1202 err = otx2_check_pf_usable(pf); 1203 if (err) 1204 goto err_free_netdev; 1205 1206 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT, 1207 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); 1208 if (err < 0) { 1209 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n", 1210 __func__, num_vec); 1211 goto err_free_netdev; 1212 } 1213 1214 /* Init PF <=> AF mailbox stuff */ 1215 err = otx2_pfaf_mbox_init(pf); 1216 if (err) 1217 goto err_free_irq_vectors; 1218 1219 /* Register mailbox interrupt */ 1220 err = otx2_register_mbox_intr(pf, true); 1221 if (err) 1222 goto err_mbox_destroy; 1223 1224 /* Request AF to attach NPA and NIX LFs to this PF. 1225 * NIX and NPA LFs are needed for this PF to function as a NIC. 1226 */ 1227 err = otx2_attach_npa_nix(pf); 1228 if (err) 1229 goto err_disable_mbox_intr; 1230 1231 err = otx2_realloc_msix_vectors(pf); 1232 if (err) 1233 goto err_detach_rsrc; 1234 1235 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues); 1236 if (err) 1237 goto err_detach_rsrc; 1238 1239 otx2_setup_dev_hw_settings(pf); 1240 1241 /* Assign default mac address */ 1242 otx2_get_mac_from_af(netdev); 1243 1244 /* NPA's pool is a stack to which SW frees buffer pointers via Aura. 1245 * HW allocates buffer pointer from stack and uses it for DMA'ing 1246 * ingress packet. In some scenarios HW can free back allocated buffer 1247 * pointers to pool. This makes it impossible for SW to maintain a 1248 * parallel list where physical addresses of buffer pointers (IOVAs) 1249 * given to HW can be saved for later reference. 1250 * 1251 * So the only way to convert Rx packet's buffer address is to use 1252 * IOMMU's iova_to_phys() handler which translates the address by 1253 * walking through the translation tables. 1254 */ 1255 pf->iommu_domain = iommu_get_domain_for_dev(dev); 1256 1257 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | 1258 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH | 1259 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6); 1260 netdev->features |= netdev->hw_features; 1261 1262 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL; 1263 1264 netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; 1265 netdev->watchdog_timeo = OTX2_TX_TIMEOUT; 1266 1267 netdev->netdev_ops = &otx2_netdev_ops; 1268 1269 /* MTU range: 64 - 9190 */ 1270 netdev->min_mtu = OTX2_MIN_MTU; 1271 netdev->max_mtu = OTX2_MAX_MTU; 1272 1273 INIT_WORK(&pf->reset_task, otx2_reset_task); 1274 1275 err = register_netdev(netdev); 1276 if (err) { 1277 dev_err(dev, "Failed to register netdevice\n"); 1278 goto err_detach_rsrc; 1279 } 1280 1281 otx2_set_ethtool_ops(netdev); 1282 1283 /* Enable link notifications */ 1284 otx2_cgx_config_linkevents(pf, true); 1285 1286 return 0; 1287 1288 err_detach_rsrc: 1289 otx2_detach_resources(&pf->mbox); 1290 err_disable_mbox_intr: 1291 otx2_disable_mbox_intr(pf); 1292 err_mbox_destroy: 1293 otx2_pfaf_mbox_destroy(pf); 1294 err_free_irq_vectors: 1295 pci_free_irq_vectors(hw->pdev); 1296 err_free_netdev: 1297 pci_set_drvdata(pdev, NULL); 1298 free_netdev(netdev); 1299 err_release_regions: 1300 pci_release_regions(pdev); 1301 return err; 1302 } 1303 1304 static void otx2_remove(struct pci_dev *pdev) 1305 { 1306 struct net_device *netdev = pci_get_drvdata(pdev); 1307 struct otx2_nic *pf; 1308 1309 if (!netdev) 1310 return; 1311 1312 pf = netdev_priv(netdev); 1313 1314 /* Disable link notifications */ 1315 otx2_cgx_config_linkevents(pf, false); 1316 1317 unregister_netdev(netdev); 1318 otx2_detach_resources(&pf->mbox); 1319 otx2_disable_mbox_intr(pf); 1320 otx2_pfaf_mbox_destroy(pf); 1321 pci_free_irq_vectors(pf->pdev); 1322 pci_set_drvdata(pdev, NULL); 1323 free_netdev(netdev); 1324 1325 pci_release_regions(pdev); 1326 } 1327 1328 static struct pci_driver otx2_pf_driver = { 1329 .name = DRV_NAME, 1330 .id_table = otx2_pf_id_table, 1331 .probe = otx2_probe, 1332 .shutdown = otx2_remove, 1333 .remove = otx2_remove, 1334 }; 1335 1336 static int __init otx2_rvupf_init_module(void) 1337 { 1338 pr_info("%s: %s\n", DRV_NAME, DRV_STRING); 1339 1340 return pci_register_driver(&otx2_pf_driver); 1341 } 1342 1343 static void __exit otx2_rvupf_cleanup_module(void) 1344 { 1345 pci_unregister_driver(&otx2_pf_driver); 1346 } 1347 1348 module_init(otx2_rvupf_init_module); 1349 module_exit(otx2_rvupf_cleanup_module); 1350