1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 32 char vmxnet3_driver_name[] = "vmxnet3"; 33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 34 35 /* 36 * PCI Device ID Table 37 * Last entry must be all 0s 38 */ 39 static const struct pci_device_id vmxnet3_pciid_table[] = { 40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 41 {0} 42 }; 43 44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 45 46 static int enable_mq = 1; 47 48 static void 49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 50 51 /* 52 * Enable/Disable the given intr 53 */ 54 static void 55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 56 { 57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 58 } 59 60 61 static void 62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 63 { 64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 65 } 66 67 68 /* 69 * Enable/Disable all intrs used by the device 70 */ 71 static void 72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 73 { 74 int i; 75 76 for (i = 0; i < adapter->intr.num_intrs; i++) 77 vmxnet3_enable_intr(adapter, i); 78 adapter->shared->devRead.intrConf.intrCtrl &= 79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 80 } 81 82 83 static void 84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 85 { 86 int i; 87 88 adapter->shared->devRead.intrConf.intrCtrl |= 89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 90 for (i = 0; i < adapter->intr.num_intrs; i++) 91 vmxnet3_disable_intr(adapter, i); 92 } 93 94 95 static void 96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 97 { 98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 99 } 100 101 102 static bool 103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 104 { 105 return tq->stopped; 106 } 107 108 109 static void 110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 111 { 112 tq->stopped = false; 113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 114 } 115 116 117 static void 118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 119 { 120 tq->stopped = false; 121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 122 } 123 124 125 static void 126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 127 { 128 tq->stopped = true; 129 tq->num_stop++; 130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 131 } 132 133 /* Check if capability is supported by UPT device or 134 * UPT is even requested 135 */ 136 bool 137 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 138 { 139 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 140 cap_supported & (1UL << cap)) { 141 return true; 142 } 143 144 return false; 145 } 146 147 148 /* 149 * Check the link state. This may start or stop the tx queue. 150 */ 151 static void 152 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 153 { 154 u32 ret; 155 int i; 156 unsigned long flags; 157 158 spin_lock_irqsave(&adapter->cmd_lock, flags); 159 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 160 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 161 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 162 163 adapter->link_speed = ret >> 16; 164 if (ret & 1) { /* Link is up. */ 165 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 166 adapter->link_speed); 167 netif_carrier_on(adapter->netdev); 168 169 if (affectTxQueue) { 170 for (i = 0; i < adapter->num_tx_queues; i++) 171 vmxnet3_tq_start(&adapter->tx_queue[i], 172 adapter); 173 } 174 } else { 175 netdev_info(adapter->netdev, "NIC Link is Down\n"); 176 netif_carrier_off(adapter->netdev); 177 178 if (affectTxQueue) { 179 for (i = 0; i < adapter->num_tx_queues; i++) 180 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 181 } 182 } 183 } 184 185 static void 186 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 187 { 188 int i; 189 unsigned long flags; 190 u32 events = le32_to_cpu(adapter->shared->ecr); 191 if (!events) 192 return; 193 194 vmxnet3_ack_events(adapter, events); 195 196 /* Check if link state has changed */ 197 if (events & VMXNET3_ECR_LINK) 198 vmxnet3_check_link(adapter, true); 199 200 /* Check if there is an error on xmit/recv queues */ 201 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 202 spin_lock_irqsave(&adapter->cmd_lock, flags); 203 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 204 VMXNET3_CMD_GET_QUEUE_STATUS); 205 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 206 207 for (i = 0; i < adapter->num_tx_queues; i++) 208 if (adapter->tqd_start[i].status.stopped) 209 dev_err(&adapter->netdev->dev, 210 "%s: tq[%d] error 0x%x\n", 211 adapter->netdev->name, i, le32_to_cpu( 212 adapter->tqd_start[i].status.error)); 213 for (i = 0; i < adapter->num_rx_queues; i++) 214 if (adapter->rqd_start[i].status.stopped) 215 dev_err(&adapter->netdev->dev, 216 "%s: rq[%d] error 0x%x\n", 217 adapter->netdev->name, i, 218 adapter->rqd_start[i].status.error); 219 220 schedule_work(&adapter->work); 221 } 222 } 223 224 #ifdef __BIG_ENDIAN_BITFIELD 225 /* 226 * The device expects the bitfields in shared structures to be written in 227 * little endian. When CPU is big endian, the following routines are used to 228 * correctly read and write into ABI. 229 * The general technique used here is : double word bitfields are defined in 230 * opposite order for big endian architecture. Then before reading them in 231 * driver the complete double word is translated using le32_to_cpu. Similarly 232 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 233 * double words into required format. 234 * In order to avoid touching bits in shared structure more than once, temporary 235 * descriptors are used. These are passed as srcDesc to following functions. 236 */ 237 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 238 struct Vmxnet3_RxDesc *dstDesc) 239 { 240 u32 *src = (u32 *)srcDesc + 2; 241 u32 *dst = (u32 *)dstDesc + 2; 242 dstDesc->addr = le64_to_cpu(srcDesc->addr); 243 *dst = le32_to_cpu(*src); 244 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 245 } 246 247 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 248 struct Vmxnet3_TxDesc *dstDesc) 249 { 250 int i; 251 u32 *src = (u32 *)(srcDesc + 1); 252 u32 *dst = (u32 *)(dstDesc + 1); 253 254 /* Working backwards so that the gen bit is set at the end. */ 255 for (i = 2; i > 0; i--) { 256 src--; 257 dst--; 258 *dst = cpu_to_le32(*src); 259 } 260 } 261 262 263 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 264 struct Vmxnet3_RxCompDesc *dstDesc) 265 { 266 int i = 0; 267 u32 *src = (u32 *)srcDesc; 268 u32 *dst = (u32 *)dstDesc; 269 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 270 *dst = le32_to_cpu(*src); 271 src++; 272 dst++; 273 } 274 } 275 276 277 /* Used to read bitfield values from double words. */ 278 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 279 { 280 u32 temp = le32_to_cpu(*bitfield); 281 u32 mask = ((1 << size) - 1) << pos; 282 temp &= mask; 283 temp >>= pos; 284 return temp; 285 } 286 287 288 289 #endif /* __BIG_ENDIAN_BITFIELD */ 290 291 #ifdef __BIG_ENDIAN_BITFIELD 292 293 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 294 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 295 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 296 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 297 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 298 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 299 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 300 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 301 VMXNET3_TCD_GEN_SIZE) 302 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 303 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 304 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 305 (dstrcd) = (tmp); \ 306 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 307 } while (0) 308 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 309 (dstrxd) = (tmp); \ 310 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 311 } while (0) 312 313 #else 314 315 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 316 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 317 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 318 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 319 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 320 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 321 322 #endif /* __BIG_ENDIAN_BITFIELD */ 323 324 325 static void 326 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 327 struct pci_dev *pdev) 328 { 329 if (tbi->map_type == VMXNET3_MAP_SINGLE) 330 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 331 DMA_TO_DEVICE); 332 else if (tbi->map_type == VMXNET3_MAP_PAGE) 333 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 334 DMA_TO_DEVICE); 335 else 336 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE); 337 338 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 339 } 340 341 342 static int 343 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 344 struct pci_dev *pdev, struct vmxnet3_adapter *adapter) 345 { 346 struct sk_buff *skb; 347 int entries = 0; 348 349 /* no out of order completion */ 350 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 351 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 352 353 skb = tq->buf_info[eop_idx].skb; 354 BUG_ON(skb == NULL); 355 tq->buf_info[eop_idx].skb = NULL; 356 357 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 358 359 while (tq->tx_ring.next2comp != eop_idx) { 360 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 361 pdev); 362 363 /* update next2comp w/o tx_lock. Since we are marking more, 364 * instead of less, tx ring entries avail, the worst case is 365 * that the tx routine incorrectly re-queues a pkt due to 366 * insufficient tx ring entries. 367 */ 368 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 369 entries++; 370 } 371 372 dev_kfree_skb_any(skb); 373 return entries; 374 } 375 376 377 static int 378 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 379 struct vmxnet3_adapter *adapter) 380 { 381 int completed = 0; 382 union Vmxnet3_GenericDesc *gdesc; 383 384 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 385 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 386 /* Prevent any &gdesc->tcd field from being (speculatively) 387 * read before (&gdesc->tcd)->gen is read. 388 */ 389 dma_rmb(); 390 391 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 392 &gdesc->tcd), tq, adapter->pdev, 393 adapter); 394 395 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 396 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 397 } 398 399 if (completed) { 400 spin_lock(&tq->tx_lock); 401 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 402 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 403 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 404 netif_carrier_ok(adapter->netdev))) { 405 vmxnet3_tq_wake(tq, adapter); 406 } 407 spin_unlock(&tq->tx_lock); 408 } 409 return completed; 410 } 411 412 413 static void 414 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 415 struct vmxnet3_adapter *adapter) 416 { 417 int i; 418 419 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 420 struct vmxnet3_tx_buf_info *tbi; 421 422 tbi = tq->buf_info + tq->tx_ring.next2comp; 423 424 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 425 if (tbi->skb) { 426 dev_kfree_skb_any(tbi->skb); 427 tbi->skb = NULL; 428 } 429 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 430 } 431 432 /* sanity check, verify all buffers are indeed unmapped and freed */ 433 for (i = 0; i < tq->tx_ring.size; i++) { 434 BUG_ON(tq->buf_info[i].skb != NULL || 435 tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 436 } 437 438 tq->tx_ring.gen = VMXNET3_INIT_GEN; 439 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 440 441 tq->comp_ring.gen = VMXNET3_INIT_GEN; 442 tq->comp_ring.next2proc = 0; 443 } 444 445 446 static void 447 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 448 struct vmxnet3_adapter *adapter) 449 { 450 if (tq->tx_ring.base) { 451 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 452 sizeof(struct Vmxnet3_TxDesc), 453 tq->tx_ring.base, tq->tx_ring.basePA); 454 tq->tx_ring.base = NULL; 455 } 456 if (tq->data_ring.base) { 457 dma_free_coherent(&adapter->pdev->dev, 458 tq->data_ring.size * tq->txdata_desc_size, 459 tq->data_ring.base, tq->data_ring.basePA); 460 tq->data_ring.base = NULL; 461 } 462 if (tq->comp_ring.base) { 463 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 464 sizeof(struct Vmxnet3_TxCompDesc), 465 tq->comp_ring.base, tq->comp_ring.basePA); 466 tq->comp_ring.base = NULL; 467 } 468 kfree(tq->buf_info); 469 tq->buf_info = NULL; 470 } 471 472 473 /* Destroy all tx queues */ 474 void 475 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 476 { 477 int i; 478 479 for (i = 0; i < adapter->num_tx_queues; i++) 480 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 481 } 482 483 484 static void 485 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 486 struct vmxnet3_adapter *adapter) 487 { 488 int i; 489 490 /* reset the tx ring contents to 0 and reset the tx ring states */ 491 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 492 sizeof(struct Vmxnet3_TxDesc)); 493 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 494 tq->tx_ring.gen = VMXNET3_INIT_GEN; 495 496 memset(tq->data_ring.base, 0, 497 tq->data_ring.size * tq->txdata_desc_size); 498 499 /* reset the tx comp ring contents to 0 and reset comp ring states */ 500 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 501 sizeof(struct Vmxnet3_TxCompDesc)); 502 tq->comp_ring.next2proc = 0; 503 tq->comp_ring.gen = VMXNET3_INIT_GEN; 504 505 /* reset the bookkeeping data */ 506 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 507 for (i = 0; i < tq->tx_ring.size; i++) 508 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 509 510 /* stats are not reset */ 511 } 512 513 514 static int 515 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 516 struct vmxnet3_adapter *adapter) 517 { 518 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 519 tq->comp_ring.base || tq->buf_info); 520 521 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 522 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 523 &tq->tx_ring.basePA, GFP_KERNEL); 524 if (!tq->tx_ring.base) { 525 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 526 goto err; 527 } 528 529 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 530 tq->data_ring.size * tq->txdata_desc_size, 531 &tq->data_ring.basePA, GFP_KERNEL); 532 if (!tq->data_ring.base) { 533 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 534 goto err; 535 } 536 537 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 538 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 539 &tq->comp_ring.basePA, GFP_KERNEL); 540 if (!tq->comp_ring.base) { 541 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 542 goto err; 543 } 544 545 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 546 GFP_KERNEL, 547 dev_to_node(&adapter->pdev->dev)); 548 if (!tq->buf_info) 549 goto err; 550 551 return 0; 552 553 err: 554 vmxnet3_tq_destroy(tq, adapter); 555 return -ENOMEM; 556 } 557 558 static void 559 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 560 { 561 int i; 562 563 for (i = 0; i < adapter->num_tx_queues; i++) 564 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 565 } 566 567 /* 568 * starting from ring->next2fill, allocate rx buffers for the given ring 569 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 570 * are allocated or allocation fails 571 */ 572 573 static int 574 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 575 int num_to_alloc, struct vmxnet3_adapter *adapter) 576 { 577 int num_allocated = 0; 578 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 579 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 580 u32 val; 581 582 while (num_allocated <= num_to_alloc) { 583 struct vmxnet3_rx_buf_info *rbi; 584 union Vmxnet3_GenericDesc *gd; 585 586 rbi = rbi_base + ring->next2fill; 587 gd = ring->base + ring->next2fill; 588 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 589 590 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 591 if (rbi->skb == NULL) { 592 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 593 rbi->len, 594 GFP_KERNEL); 595 if (unlikely(rbi->skb == NULL)) { 596 rq->stats.rx_buf_alloc_failure++; 597 break; 598 } 599 600 rbi->dma_addr = dma_map_single( 601 &adapter->pdev->dev, 602 rbi->skb->data, rbi->len, 603 DMA_FROM_DEVICE); 604 if (dma_mapping_error(&adapter->pdev->dev, 605 rbi->dma_addr)) { 606 dev_kfree_skb_any(rbi->skb); 607 rbi->skb = NULL; 608 rq->stats.rx_buf_alloc_failure++; 609 break; 610 } 611 } else { 612 /* rx buffer skipped by the device */ 613 } 614 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 615 } else { 616 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 617 rbi->len != PAGE_SIZE); 618 619 if (rbi->page == NULL) { 620 rbi->page = alloc_page(GFP_ATOMIC); 621 if (unlikely(rbi->page == NULL)) { 622 rq->stats.rx_buf_alloc_failure++; 623 break; 624 } 625 rbi->dma_addr = dma_map_page( 626 &adapter->pdev->dev, 627 rbi->page, 0, PAGE_SIZE, 628 DMA_FROM_DEVICE); 629 if (dma_mapping_error(&adapter->pdev->dev, 630 rbi->dma_addr)) { 631 put_page(rbi->page); 632 rbi->page = NULL; 633 rq->stats.rx_buf_alloc_failure++; 634 break; 635 } 636 } else { 637 /* rx buffers skipped by the device */ 638 } 639 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 640 } 641 642 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 643 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 644 | val | rbi->len); 645 646 /* Fill the last buffer but dont mark it ready, or else the 647 * device will think that the queue is full */ 648 if (num_allocated == num_to_alloc) { 649 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 650 break; 651 } 652 653 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 654 num_allocated++; 655 vmxnet3_cmd_ring_adv_next2fill(ring); 656 } 657 658 netdev_dbg(adapter->netdev, 659 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 660 num_allocated, ring->next2fill, ring->next2comp); 661 662 /* so that the device can distinguish a full ring and an empty ring */ 663 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 664 665 return num_allocated; 666 } 667 668 669 static void 670 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 671 struct vmxnet3_rx_buf_info *rbi) 672 { 673 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 674 675 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 676 677 __skb_frag_set_page(frag, rbi->page); 678 skb_frag_off_set(frag, 0); 679 skb_frag_size_set(frag, rcd->len); 680 skb->data_len += rcd->len; 681 skb->truesize += PAGE_SIZE; 682 skb_shinfo(skb)->nr_frags++; 683 } 684 685 686 static int 687 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 688 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 689 struct vmxnet3_adapter *adapter) 690 { 691 u32 dw2, len; 692 unsigned long buf_offset; 693 int i; 694 union Vmxnet3_GenericDesc *gdesc; 695 struct vmxnet3_tx_buf_info *tbi = NULL; 696 697 BUG_ON(ctx->copy_size > skb_headlen(skb)); 698 699 /* use the previous gen bit for the SOP desc */ 700 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 701 702 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 703 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 704 705 /* no need to map the buffer if headers are copied */ 706 if (ctx->copy_size) { 707 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 708 tq->tx_ring.next2fill * 709 tq->txdata_desc_size); 710 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 711 ctx->sop_txd->dword[3] = 0; 712 713 tbi = tq->buf_info + tq->tx_ring.next2fill; 714 tbi->map_type = VMXNET3_MAP_NONE; 715 716 netdev_dbg(adapter->netdev, 717 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 718 tq->tx_ring.next2fill, 719 le64_to_cpu(ctx->sop_txd->txd.addr), 720 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 721 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 722 723 /* use the right gen for non-SOP desc */ 724 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 725 } 726 727 /* linear part can use multiple tx desc if it's big */ 728 len = skb_headlen(skb) - ctx->copy_size; 729 buf_offset = ctx->copy_size; 730 while (len) { 731 u32 buf_size; 732 733 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 734 buf_size = len; 735 dw2 |= len; 736 } else { 737 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 738 /* spec says that for TxDesc.len, 0 == 2^14 */ 739 } 740 741 tbi = tq->buf_info + tq->tx_ring.next2fill; 742 tbi->map_type = VMXNET3_MAP_SINGLE; 743 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 744 skb->data + buf_offset, buf_size, 745 DMA_TO_DEVICE); 746 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 747 return -EFAULT; 748 749 tbi->len = buf_size; 750 751 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 752 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 753 754 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 755 gdesc->dword[2] = cpu_to_le32(dw2); 756 gdesc->dword[3] = 0; 757 758 netdev_dbg(adapter->netdev, 759 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 760 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 761 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 762 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 763 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 764 765 len -= buf_size; 766 buf_offset += buf_size; 767 } 768 769 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 770 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 771 u32 buf_size; 772 773 buf_offset = 0; 774 len = skb_frag_size(frag); 775 while (len) { 776 tbi = tq->buf_info + tq->tx_ring.next2fill; 777 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 778 buf_size = len; 779 dw2 |= len; 780 } else { 781 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 782 /* spec says that for TxDesc.len, 0 == 2^14 */ 783 } 784 tbi->map_type = VMXNET3_MAP_PAGE; 785 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 786 buf_offset, buf_size, 787 DMA_TO_DEVICE); 788 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 789 return -EFAULT; 790 791 tbi->len = buf_size; 792 793 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 794 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 795 796 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 797 gdesc->dword[2] = cpu_to_le32(dw2); 798 gdesc->dword[3] = 0; 799 800 netdev_dbg(adapter->netdev, 801 "txd[%u]: 0x%llx %u %u\n", 802 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 803 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 804 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 805 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 806 807 len -= buf_size; 808 buf_offset += buf_size; 809 } 810 } 811 812 ctx->eop_txd = gdesc; 813 814 /* set the last buf_info for the pkt */ 815 tbi->skb = skb; 816 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 817 818 return 0; 819 } 820 821 822 /* Init all tx queues */ 823 static void 824 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 825 { 826 int i; 827 828 for (i = 0; i < adapter->num_tx_queues; i++) 829 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 830 } 831 832 833 /* 834 * parse relevant protocol headers: 835 * For a tso pkt, relevant headers are L2/3/4 including options 836 * For a pkt requesting csum offloading, they are L2/3 and may include L4 837 * if it's a TCP/UDP pkt 838 * 839 * Returns: 840 * -1: error happens during parsing 841 * 0: protocol headers parsed, but too big to be copied 842 * 1: protocol headers parsed and copied 843 * 844 * Other effects: 845 * 1. related *ctx fields are updated. 846 * 2. ctx->copy_size is # of bytes copied 847 * 3. the portion to be copied is guaranteed to be in the linear part 848 * 849 */ 850 static int 851 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 852 struct vmxnet3_tx_ctx *ctx, 853 struct vmxnet3_adapter *adapter) 854 { 855 u8 protocol = 0; 856 857 if (ctx->mss) { /* TSO */ 858 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 859 ctx->l4_offset = skb_inner_transport_offset(skb); 860 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 861 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 862 } else { 863 ctx->l4_offset = skb_transport_offset(skb); 864 ctx->l4_hdr_size = tcp_hdrlen(skb); 865 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 866 } 867 } else { 868 if (skb->ip_summed == CHECKSUM_PARTIAL) { 869 /* For encap packets, skb_checksum_start_offset refers 870 * to inner L4 offset. Thus, below works for encap as 871 * well as non-encap case 872 */ 873 ctx->l4_offset = skb_checksum_start_offset(skb); 874 875 if (VMXNET3_VERSION_GE_4(adapter) && 876 skb->encapsulation) { 877 struct iphdr *iph = inner_ip_hdr(skb); 878 879 if (iph->version == 4) { 880 protocol = iph->protocol; 881 } else { 882 const struct ipv6hdr *ipv6h; 883 884 ipv6h = inner_ipv6_hdr(skb); 885 protocol = ipv6h->nexthdr; 886 } 887 } else { 888 if (ctx->ipv4) { 889 const struct iphdr *iph = ip_hdr(skb); 890 891 protocol = iph->protocol; 892 } else if (ctx->ipv6) { 893 const struct ipv6hdr *ipv6h; 894 895 ipv6h = ipv6_hdr(skb); 896 protocol = ipv6h->nexthdr; 897 } 898 } 899 900 switch (protocol) { 901 case IPPROTO_TCP: 902 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 903 tcp_hdrlen(skb); 904 break; 905 case IPPROTO_UDP: 906 ctx->l4_hdr_size = sizeof(struct udphdr); 907 break; 908 default: 909 ctx->l4_hdr_size = 0; 910 break; 911 } 912 913 ctx->copy_size = min(ctx->l4_offset + 914 ctx->l4_hdr_size, skb->len); 915 } else { 916 ctx->l4_offset = 0; 917 ctx->l4_hdr_size = 0; 918 /* copy as much as allowed */ 919 ctx->copy_size = min_t(unsigned int, 920 tq->txdata_desc_size, 921 skb_headlen(skb)); 922 } 923 924 if (skb->len <= VMXNET3_HDR_COPY_SIZE) 925 ctx->copy_size = skb->len; 926 927 /* make sure headers are accessible directly */ 928 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 929 goto err; 930 } 931 932 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 933 tq->stats.oversized_hdr++; 934 ctx->copy_size = 0; 935 return 0; 936 } 937 938 return 1; 939 err: 940 return -1; 941 } 942 943 /* 944 * copy relevant protocol headers to the transmit ring: 945 * For a tso pkt, relevant headers are L2/3/4 including options 946 * For a pkt requesting csum offloading, they are L2/3 and may include L4 947 * if it's a TCP/UDP pkt 948 * 949 * 950 * Note that this requires that vmxnet3_parse_hdr be called first to set the 951 * appropriate bits in ctx first 952 */ 953 static void 954 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 955 struct vmxnet3_tx_ctx *ctx, 956 struct vmxnet3_adapter *adapter) 957 { 958 struct Vmxnet3_TxDataDesc *tdd; 959 960 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 961 tq->tx_ring.next2fill * 962 tq->txdata_desc_size); 963 964 memcpy(tdd->data, skb->data, ctx->copy_size); 965 netdev_dbg(adapter->netdev, 966 "copy %u bytes to dataRing[%u]\n", 967 ctx->copy_size, tq->tx_ring.next2fill); 968 } 969 970 971 static void 972 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 973 struct vmxnet3_tx_ctx *ctx) 974 { 975 struct tcphdr *tcph = inner_tcp_hdr(skb); 976 struct iphdr *iph = inner_ip_hdr(skb); 977 978 if (iph->version == 4) { 979 iph->check = 0; 980 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 981 IPPROTO_TCP, 0); 982 } else { 983 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 984 985 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 986 IPPROTO_TCP, 0); 987 } 988 } 989 990 static void 991 vmxnet3_prepare_tso(struct sk_buff *skb, 992 struct vmxnet3_tx_ctx *ctx) 993 { 994 struct tcphdr *tcph = tcp_hdr(skb); 995 996 if (ctx->ipv4) { 997 struct iphdr *iph = ip_hdr(skb); 998 999 iph->check = 0; 1000 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1001 IPPROTO_TCP, 0); 1002 } else if (ctx->ipv6) { 1003 tcp_v6_gso_csum_prep(skb); 1004 } 1005 } 1006 1007 static int txd_estimate(const struct sk_buff *skb) 1008 { 1009 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1010 int i; 1011 1012 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1013 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1014 1015 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1016 } 1017 return count; 1018 } 1019 1020 /* 1021 * Transmits a pkt thru a given tq 1022 * Returns: 1023 * NETDEV_TX_OK: descriptors are setup successfully 1024 * NETDEV_TX_OK: error occurred, the pkt is dropped 1025 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1026 * 1027 * Side-effects: 1028 * 1. tx ring may be changed 1029 * 2. tq stats may be updated accordingly 1030 * 3. shared->txNumDeferred may be updated 1031 */ 1032 1033 static int 1034 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1035 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1036 { 1037 int ret; 1038 u32 count; 1039 int num_pkts; 1040 int tx_num_deferred; 1041 unsigned long flags; 1042 struct vmxnet3_tx_ctx ctx; 1043 union Vmxnet3_GenericDesc *gdesc; 1044 #ifdef __BIG_ENDIAN_BITFIELD 1045 /* Use temporary descriptor to avoid touching bits multiple times */ 1046 union Vmxnet3_GenericDesc tempTxDesc; 1047 #endif 1048 1049 count = txd_estimate(skb); 1050 1051 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1052 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1053 1054 ctx.mss = skb_shinfo(skb)->gso_size; 1055 if (ctx.mss) { 1056 if (skb_header_cloned(skb)) { 1057 if (unlikely(pskb_expand_head(skb, 0, 0, 1058 GFP_ATOMIC) != 0)) { 1059 tq->stats.drop_tso++; 1060 goto drop_pkt; 1061 } 1062 tq->stats.copy_skb_header++; 1063 } 1064 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1065 /* tso pkts must not use more than 1066 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1067 */ 1068 if (skb_linearize(skb) != 0) { 1069 tq->stats.drop_too_many_frags++; 1070 goto drop_pkt; 1071 } 1072 tq->stats.linearized++; 1073 1074 /* recalculate the # of descriptors to use */ 1075 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1076 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1077 tq->stats.drop_too_many_frags++; 1078 goto drop_pkt; 1079 } 1080 } 1081 if (skb->encapsulation) { 1082 vmxnet3_prepare_inner_tso(skb, &ctx); 1083 } else { 1084 vmxnet3_prepare_tso(skb, &ctx); 1085 } 1086 } else { 1087 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1088 1089 /* non-tso pkts must not use more than 1090 * VMXNET3_MAX_TXD_PER_PKT entries 1091 */ 1092 if (skb_linearize(skb) != 0) { 1093 tq->stats.drop_too_many_frags++; 1094 goto drop_pkt; 1095 } 1096 tq->stats.linearized++; 1097 1098 /* recalculate the # of descriptors to use */ 1099 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1100 } 1101 } 1102 1103 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1104 if (ret >= 0) { 1105 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1106 /* hdrs parsed, check against other limits */ 1107 if (ctx.mss) { 1108 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1109 VMXNET3_MAX_TX_BUF_SIZE)) { 1110 tq->stats.drop_oversized_hdr++; 1111 goto drop_pkt; 1112 } 1113 } else { 1114 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1115 if (unlikely(ctx.l4_offset + 1116 skb->csum_offset > 1117 VMXNET3_MAX_CSUM_OFFSET)) { 1118 tq->stats.drop_oversized_hdr++; 1119 goto drop_pkt; 1120 } 1121 } 1122 } 1123 } else { 1124 tq->stats.drop_hdr_inspect_err++; 1125 goto drop_pkt; 1126 } 1127 1128 spin_lock_irqsave(&tq->tx_lock, flags); 1129 1130 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1131 tq->stats.tx_ring_full++; 1132 netdev_dbg(adapter->netdev, 1133 "tx queue stopped on %s, next2comp %u" 1134 " next2fill %u\n", adapter->netdev->name, 1135 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1136 1137 vmxnet3_tq_stop(tq, adapter); 1138 spin_unlock_irqrestore(&tq->tx_lock, flags); 1139 return NETDEV_TX_BUSY; 1140 } 1141 1142 1143 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1144 1145 /* fill tx descs related to addr & len */ 1146 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1147 goto unlock_drop_pkt; 1148 1149 /* setup the EOP desc */ 1150 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1151 1152 /* setup the SOP desc */ 1153 #ifdef __BIG_ENDIAN_BITFIELD 1154 gdesc = &tempTxDesc; 1155 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1156 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1157 #else 1158 gdesc = ctx.sop_txd; 1159 #endif 1160 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1161 if (ctx.mss) { 1162 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1163 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1164 if (VMXNET3_VERSION_GE_7(adapter)) { 1165 gdesc->txd.om = VMXNET3_OM_TSO; 1166 gdesc->txd.ext1 = 1; 1167 } else { 1168 gdesc->txd.om = VMXNET3_OM_ENCAP; 1169 } 1170 gdesc->txd.msscof = ctx.mss; 1171 1172 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1173 gdesc->txd.oco = 1; 1174 } else { 1175 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1176 gdesc->txd.om = VMXNET3_OM_TSO; 1177 gdesc->txd.msscof = ctx.mss; 1178 } 1179 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1180 } else { 1181 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1182 if (VMXNET3_VERSION_GE_4(adapter) && 1183 skb->encapsulation) { 1184 gdesc->txd.hlen = ctx.l4_offset + 1185 ctx.l4_hdr_size; 1186 if (VMXNET3_VERSION_GE_7(adapter)) { 1187 gdesc->txd.om = VMXNET3_OM_CSUM; 1188 gdesc->txd.msscof = ctx.l4_offset + 1189 skb->csum_offset; 1190 gdesc->txd.ext1 = 1; 1191 } else { 1192 gdesc->txd.om = VMXNET3_OM_ENCAP; 1193 gdesc->txd.msscof = 0; /* Reserved */ 1194 } 1195 } else { 1196 gdesc->txd.hlen = ctx.l4_offset; 1197 gdesc->txd.om = VMXNET3_OM_CSUM; 1198 gdesc->txd.msscof = ctx.l4_offset + 1199 skb->csum_offset; 1200 } 1201 } else { 1202 gdesc->txd.om = 0; 1203 gdesc->txd.msscof = 0; 1204 } 1205 num_pkts = 1; 1206 } 1207 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1208 tx_num_deferred += num_pkts; 1209 1210 if (skb_vlan_tag_present(skb)) { 1211 gdesc->txd.ti = 1; 1212 gdesc->txd.tci = skb_vlan_tag_get(skb); 1213 } 1214 1215 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1216 * all other writes to &gdesc->txd. 1217 */ 1218 dma_wmb(); 1219 1220 /* finally flips the GEN bit of the SOP desc. */ 1221 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1222 VMXNET3_TXD_GEN); 1223 #ifdef __BIG_ENDIAN_BITFIELD 1224 /* Finished updating in bitfields of Tx Desc, so write them in original 1225 * place. 1226 */ 1227 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1228 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1229 gdesc = ctx.sop_txd; 1230 #endif 1231 netdev_dbg(adapter->netdev, 1232 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1233 (u32)(ctx.sop_txd - 1234 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1235 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1236 1237 spin_unlock_irqrestore(&tq->tx_lock, flags); 1238 1239 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1240 tq->shared->txNumDeferred = 0; 1241 VMXNET3_WRITE_BAR0_REG(adapter, 1242 adapter->tx_prod_offset + tq->qid * 8, 1243 tq->tx_ring.next2fill); 1244 } 1245 1246 return NETDEV_TX_OK; 1247 1248 unlock_drop_pkt: 1249 spin_unlock_irqrestore(&tq->tx_lock, flags); 1250 drop_pkt: 1251 tq->stats.drop_total++; 1252 dev_kfree_skb_any(skb); 1253 return NETDEV_TX_OK; 1254 } 1255 1256 1257 static netdev_tx_t 1258 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1259 { 1260 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1261 1262 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1263 return vmxnet3_tq_xmit(skb, 1264 &adapter->tx_queue[skb->queue_mapping], 1265 adapter, netdev); 1266 } 1267 1268 1269 static void 1270 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1271 struct sk_buff *skb, 1272 union Vmxnet3_GenericDesc *gdesc) 1273 { 1274 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1275 if (gdesc->rcd.v4 && 1276 (le32_to_cpu(gdesc->dword[3]) & 1277 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1278 skb->ip_summed = CHECKSUM_UNNECESSARY; 1279 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1280 !(le32_to_cpu(gdesc->dword[0]) & 1281 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1282 WARN_ON_ONCE(gdesc->rcd.frg && 1283 !(le32_to_cpu(gdesc->dword[0]) & 1284 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1285 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1286 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1287 skb->ip_summed = CHECKSUM_UNNECESSARY; 1288 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1289 !(le32_to_cpu(gdesc->dword[0]) & 1290 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1291 WARN_ON_ONCE(gdesc->rcd.frg && 1292 !(le32_to_cpu(gdesc->dword[0]) & 1293 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1294 } else { 1295 if (gdesc->rcd.csum) { 1296 skb->csum = htons(gdesc->rcd.csum); 1297 skb->ip_summed = CHECKSUM_PARTIAL; 1298 } else { 1299 skb_checksum_none_assert(skb); 1300 } 1301 } 1302 } else { 1303 skb_checksum_none_assert(skb); 1304 } 1305 } 1306 1307 1308 static void 1309 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1310 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1311 { 1312 rq->stats.drop_err++; 1313 if (!rcd->fcs) 1314 rq->stats.drop_fcs++; 1315 1316 rq->stats.drop_total++; 1317 1318 /* 1319 * We do not unmap and chain the rx buffer to the skb. 1320 * We basically pretend this buffer is not used and will be recycled 1321 * by vmxnet3_rq_alloc_rx_buf() 1322 */ 1323 1324 /* 1325 * ctx->skb may be NULL if this is the first and the only one 1326 * desc for the pkt 1327 */ 1328 if (ctx->skb) 1329 dev_kfree_skb_irq(ctx->skb); 1330 1331 ctx->skb = NULL; 1332 } 1333 1334 1335 static u32 1336 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1337 union Vmxnet3_GenericDesc *gdesc) 1338 { 1339 u32 hlen, maplen; 1340 union { 1341 void *ptr; 1342 struct ethhdr *eth; 1343 struct vlan_ethhdr *veth; 1344 struct iphdr *ipv4; 1345 struct ipv6hdr *ipv6; 1346 struct tcphdr *tcp; 1347 } hdr; 1348 BUG_ON(gdesc->rcd.tcp == 0); 1349 1350 maplen = skb_headlen(skb); 1351 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1352 return 0; 1353 1354 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1355 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1356 hlen = sizeof(struct vlan_ethhdr); 1357 else 1358 hlen = sizeof(struct ethhdr); 1359 1360 hdr.eth = eth_hdr(skb); 1361 if (gdesc->rcd.v4) { 1362 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1363 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1364 hdr.ptr += hlen; 1365 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1366 hlen = hdr.ipv4->ihl << 2; 1367 hdr.ptr += hdr.ipv4->ihl << 2; 1368 } else if (gdesc->rcd.v6) { 1369 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1370 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1371 hdr.ptr += hlen; 1372 /* Use an estimated value, since we also need to handle 1373 * TSO case. 1374 */ 1375 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1376 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1377 hlen = sizeof(struct ipv6hdr); 1378 hdr.ptr += sizeof(struct ipv6hdr); 1379 } else { 1380 /* Non-IP pkt, dont estimate header length */ 1381 return 0; 1382 } 1383 1384 if (hlen + sizeof(struct tcphdr) > maplen) 1385 return 0; 1386 1387 return (hlen + (hdr.tcp->doff << 2)); 1388 } 1389 1390 static int 1391 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1392 struct vmxnet3_adapter *adapter, int quota) 1393 { 1394 u32 rxprod_reg[2] = { 1395 adapter->rx_prod_offset, adapter->rx_prod2_offset 1396 }; 1397 u32 num_pkts = 0; 1398 bool skip_page_frags = false; 1399 struct Vmxnet3_RxCompDesc *rcd; 1400 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1401 u16 segCnt = 0, mss = 0; 1402 int comp_offset, fill_offset; 1403 #ifdef __BIG_ENDIAN_BITFIELD 1404 struct Vmxnet3_RxDesc rxCmdDesc; 1405 struct Vmxnet3_RxCompDesc rxComp; 1406 #endif 1407 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1408 &rxComp); 1409 while (rcd->gen == rq->comp_ring.gen) { 1410 struct vmxnet3_rx_buf_info *rbi; 1411 struct sk_buff *skb, *new_skb = NULL; 1412 struct page *new_page = NULL; 1413 dma_addr_t new_dma_addr; 1414 int num_to_alloc; 1415 struct Vmxnet3_RxDesc *rxd; 1416 u32 idx, ring_idx; 1417 struct vmxnet3_cmd_ring *ring = NULL; 1418 if (num_pkts >= quota) { 1419 /* we may stop even before we see the EOP desc of 1420 * the current pkt 1421 */ 1422 break; 1423 } 1424 1425 /* Prevent any rcd field from being (speculatively) read before 1426 * rcd->gen is read. 1427 */ 1428 dma_rmb(); 1429 1430 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1431 rcd->rqID != rq->dataRingQid); 1432 idx = rcd->rxdIdx; 1433 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1434 ring = rq->rx_ring + ring_idx; 1435 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1436 &rxCmdDesc); 1437 rbi = rq->buf_info[ring_idx] + idx; 1438 1439 BUG_ON(rxd->addr != rbi->dma_addr || 1440 rxd->len != rbi->len); 1441 1442 if (unlikely(rcd->eop && rcd->err)) { 1443 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1444 goto rcd_done; 1445 } 1446 1447 if (rcd->sop) { /* first buf of the pkt */ 1448 bool rxDataRingUsed; 1449 u16 len; 1450 1451 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1452 (rcd->rqID != rq->qid && 1453 rcd->rqID != rq->dataRingQid)); 1454 1455 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB); 1456 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1457 1458 if (unlikely(rcd->len == 0)) { 1459 /* Pretend the rx buffer is skipped. */ 1460 BUG_ON(!(rcd->sop && rcd->eop)); 1461 netdev_dbg(adapter->netdev, 1462 "rxRing[%u][%u] 0 length\n", 1463 ring_idx, idx); 1464 goto rcd_done; 1465 } 1466 1467 skip_page_frags = false; 1468 ctx->skb = rbi->skb; 1469 1470 rxDataRingUsed = 1471 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1472 len = rxDataRingUsed ? rcd->len : rbi->len; 1473 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1474 len); 1475 if (new_skb == NULL) { 1476 /* Skb allocation failed, do not handover this 1477 * skb to stack. Reuse it. Drop the existing pkt 1478 */ 1479 rq->stats.rx_buf_alloc_failure++; 1480 ctx->skb = NULL; 1481 rq->stats.drop_total++; 1482 skip_page_frags = true; 1483 goto rcd_done; 1484 } 1485 1486 if (rxDataRingUsed) { 1487 size_t sz; 1488 1489 BUG_ON(rcd->len > rq->data_ring.desc_size); 1490 1491 ctx->skb = new_skb; 1492 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1493 memcpy(new_skb->data, 1494 &rq->data_ring.base[sz], rcd->len); 1495 } else { 1496 ctx->skb = rbi->skb; 1497 1498 new_dma_addr = 1499 dma_map_single(&adapter->pdev->dev, 1500 new_skb->data, rbi->len, 1501 DMA_FROM_DEVICE); 1502 if (dma_mapping_error(&adapter->pdev->dev, 1503 new_dma_addr)) { 1504 dev_kfree_skb(new_skb); 1505 /* Skb allocation failed, do not 1506 * handover this skb to stack. Reuse 1507 * it. Drop the existing pkt. 1508 */ 1509 rq->stats.rx_buf_alloc_failure++; 1510 ctx->skb = NULL; 1511 rq->stats.drop_total++; 1512 skip_page_frags = true; 1513 goto rcd_done; 1514 } 1515 1516 dma_unmap_single(&adapter->pdev->dev, 1517 rbi->dma_addr, 1518 rbi->len, 1519 DMA_FROM_DEVICE); 1520 1521 /* Immediate refill */ 1522 rbi->skb = new_skb; 1523 rbi->dma_addr = new_dma_addr; 1524 rxd->addr = cpu_to_le64(rbi->dma_addr); 1525 rxd->len = rbi->len; 1526 } 1527 1528 #ifdef VMXNET3_RSS 1529 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1530 (adapter->netdev->features & NETIF_F_RXHASH)) { 1531 enum pkt_hash_types hash_type; 1532 1533 switch (rcd->rssType) { 1534 case VMXNET3_RCD_RSS_TYPE_IPV4: 1535 case VMXNET3_RCD_RSS_TYPE_IPV6: 1536 hash_type = PKT_HASH_TYPE_L3; 1537 break; 1538 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1539 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1540 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1541 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1542 hash_type = PKT_HASH_TYPE_L4; 1543 break; 1544 default: 1545 hash_type = PKT_HASH_TYPE_L3; 1546 break; 1547 } 1548 skb_set_hash(ctx->skb, 1549 le32_to_cpu(rcd->rssHash), 1550 hash_type); 1551 } 1552 #endif 1553 skb_record_rx_queue(ctx->skb, rq->qid); 1554 skb_put(ctx->skb, rcd->len); 1555 1556 if (VMXNET3_VERSION_GE_2(adapter) && 1557 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1558 struct Vmxnet3_RxCompDescExt *rcdlro; 1559 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1560 1561 segCnt = rcdlro->segCnt; 1562 WARN_ON_ONCE(segCnt == 0); 1563 mss = rcdlro->mss; 1564 if (unlikely(segCnt <= 1)) 1565 segCnt = 0; 1566 } else { 1567 segCnt = 0; 1568 } 1569 } else { 1570 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1571 1572 /* non SOP buffer must be type 1 in most cases */ 1573 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1574 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1575 1576 /* If an sop buffer was dropped, skip all 1577 * following non-sop fragments. They will be reused. 1578 */ 1579 if (skip_page_frags) 1580 goto rcd_done; 1581 1582 if (rcd->len) { 1583 new_page = alloc_page(GFP_ATOMIC); 1584 /* Replacement page frag could not be allocated. 1585 * Reuse this page. Drop the pkt and free the 1586 * skb which contained this page as a frag. Skip 1587 * processing all the following non-sop frags. 1588 */ 1589 if (unlikely(!new_page)) { 1590 rq->stats.rx_buf_alloc_failure++; 1591 dev_kfree_skb(ctx->skb); 1592 ctx->skb = NULL; 1593 skip_page_frags = true; 1594 goto rcd_done; 1595 } 1596 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1597 new_page, 1598 0, PAGE_SIZE, 1599 DMA_FROM_DEVICE); 1600 if (dma_mapping_error(&adapter->pdev->dev, 1601 new_dma_addr)) { 1602 put_page(new_page); 1603 rq->stats.rx_buf_alloc_failure++; 1604 dev_kfree_skb(ctx->skb); 1605 ctx->skb = NULL; 1606 skip_page_frags = true; 1607 goto rcd_done; 1608 } 1609 1610 dma_unmap_page(&adapter->pdev->dev, 1611 rbi->dma_addr, rbi->len, 1612 DMA_FROM_DEVICE); 1613 1614 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1615 1616 /* Immediate refill */ 1617 rbi->page = new_page; 1618 rbi->dma_addr = new_dma_addr; 1619 rxd->addr = cpu_to_le64(rbi->dma_addr); 1620 rxd->len = rbi->len; 1621 } 1622 } 1623 1624 1625 skb = ctx->skb; 1626 if (rcd->eop) { 1627 u32 mtu = adapter->netdev->mtu; 1628 skb->len += skb->data_len; 1629 1630 vmxnet3_rx_csum(adapter, skb, 1631 (union Vmxnet3_GenericDesc *)rcd); 1632 skb->protocol = eth_type_trans(skb, adapter->netdev); 1633 if (!rcd->tcp || 1634 !(adapter->netdev->features & NETIF_F_LRO)) 1635 goto not_lro; 1636 1637 if (segCnt != 0 && mss != 0) { 1638 skb_shinfo(skb)->gso_type = rcd->v4 ? 1639 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1640 skb_shinfo(skb)->gso_size = mss; 1641 skb_shinfo(skb)->gso_segs = segCnt; 1642 } else if (segCnt != 0 || skb->len > mtu) { 1643 u32 hlen; 1644 1645 hlen = vmxnet3_get_hdr_len(adapter, skb, 1646 (union Vmxnet3_GenericDesc *)rcd); 1647 if (hlen == 0) 1648 goto not_lro; 1649 1650 skb_shinfo(skb)->gso_type = 1651 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1652 if (segCnt != 0) { 1653 skb_shinfo(skb)->gso_segs = segCnt; 1654 skb_shinfo(skb)->gso_size = 1655 DIV_ROUND_UP(skb->len - 1656 hlen, segCnt); 1657 } else { 1658 skb_shinfo(skb)->gso_size = mtu - hlen; 1659 } 1660 } 1661 not_lro: 1662 if (unlikely(rcd->ts)) 1663 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1664 1665 if (adapter->netdev->features & NETIF_F_LRO) 1666 netif_receive_skb(skb); 1667 else 1668 napi_gro_receive(&rq->napi, skb); 1669 1670 ctx->skb = NULL; 1671 num_pkts++; 1672 } 1673 1674 rcd_done: 1675 /* device may have skipped some rx descs */ 1676 ring = rq->rx_ring + ring_idx; 1677 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1678 1679 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1680 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1681 idx - ring->next2fill - 1; 1682 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1683 ring->next2comp = idx; 1684 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1685 1686 /* Ensure that the writes to rxd->gen bits will be observed 1687 * after all other writes to rxd objects. 1688 */ 1689 dma_wmb(); 1690 1691 while (num_to_alloc) { 1692 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1693 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1694 goto refill_buf; 1695 if (ring_idx == 0) { 1696 /* ring0 Type1 buffers can get skipped; re-fill them */ 1697 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1698 goto refill_buf; 1699 } 1700 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1701 refill_buf: 1702 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1703 &rxCmdDesc); 1704 WARN_ON(!rxd->addr); 1705 1706 /* Recv desc is ready to be used by the device */ 1707 rxd->gen = ring->gen; 1708 vmxnet3_cmd_ring_adv_next2fill(ring); 1709 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1710 num_to_alloc--; 1711 } else { 1712 /* rx completion hasn't occurred */ 1713 ring->isOutOfOrder = 1; 1714 break; 1715 } 1716 } 1717 1718 if (num_to_alloc == 0) { 1719 ring->isOutOfOrder = 0; 1720 } 1721 1722 /* if needed, update the register */ 1723 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1724 VMXNET3_WRITE_BAR0_REG(adapter, 1725 rxprod_reg[ring_idx] + rq->qid * 8, 1726 ring->next2fill); 1727 } 1728 1729 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1730 vmxnet3_getRxComp(rcd, 1731 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1732 } 1733 1734 return num_pkts; 1735 } 1736 1737 1738 static void 1739 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1740 struct vmxnet3_adapter *adapter) 1741 { 1742 u32 i, ring_idx; 1743 struct Vmxnet3_RxDesc *rxd; 1744 1745 /* ring has already been cleaned up */ 1746 if (!rq->rx_ring[0].base) 1747 return; 1748 1749 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1750 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 1751 #ifdef __BIG_ENDIAN_BITFIELD 1752 struct Vmxnet3_RxDesc rxDesc; 1753 #endif 1754 vmxnet3_getRxDesc(rxd, 1755 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 1756 1757 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1758 rq->buf_info[ring_idx][i].skb) { 1759 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 1760 rxd->len, DMA_FROM_DEVICE); 1761 dev_kfree_skb(rq->buf_info[ring_idx][i].skb); 1762 rq->buf_info[ring_idx][i].skb = NULL; 1763 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 1764 rq->buf_info[ring_idx][i].page) { 1765 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 1766 rxd->len, DMA_FROM_DEVICE); 1767 put_page(rq->buf_info[ring_idx][i].page); 1768 rq->buf_info[ring_idx][i].page = NULL; 1769 } 1770 } 1771 1772 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 1773 rq->rx_ring[ring_idx].next2fill = 1774 rq->rx_ring[ring_idx].next2comp = 0; 1775 } 1776 1777 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1778 rq->comp_ring.next2proc = 0; 1779 } 1780 1781 1782 static void 1783 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 1784 { 1785 int i; 1786 1787 for (i = 0; i < adapter->num_rx_queues; i++) 1788 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 1789 } 1790 1791 1792 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1793 struct vmxnet3_adapter *adapter) 1794 { 1795 int i; 1796 int j; 1797 1798 /* all rx buffers must have already been freed */ 1799 for (i = 0; i < 2; i++) { 1800 if (rq->buf_info[i]) { 1801 for (j = 0; j < rq->rx_ring[i].size; j++) 1802 BUG_ON(rq->buf_info[i][j].page != NULL); 1803 } 1804 } 1805 1806 1807 for (i = 0; i < 2; i++) { 1808 if (rq->rx_ring[i].base) { 1809 dma_free_coherent(&adapter->pdev->dev, 1810 rq->rx_ring[i].size 1811 * sizeof(struct Vmxnet3_RxDesc), 1812 rq->rx_ring[i].base, 1813 rq->rx_ring[i].basePA); 1814 rq->rx_ring[i].base = NULL; 1815 } 1816 } 1817 1818 if (rq->data_ring.base) { 1819 dma_free_coherent(&adapter->pdev->dev, 1820 rq->rx_ring[0].size * rq->data_ring.desc_size, 1821 rq->data_ring.base, rq->data_ring.basePA); 1822 rq->data_ring.base = NULL; 1823 } 1824 1825 if (rq->comp_ring.base) { 1826 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 1827 * sizeof(struct Vmxnet3_RxCompDesc), 1828 rq->comp_ring.base, rq->comp_ring.basePA); 1829 rq->comp_ring.base = NULL; 1830 } 1831 1832 kfree(rq->buf_info[0]); 1833 rq->buf_info[0] = NULL; 1834 rq->buf_info[1] = NULL; 1835 } 1836 1837 static void 1838 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 1839 { 1840 int i; 1841 1842 for (i = 0; i < adapter->num_rx_queues; i++) { 1843 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 1844 1845 if (rq->data_ring.base) { 1846 dma_free_coherent(&adapter->pdev->dev, 1847 (rq->rx_ring[0].size * 1848 rq->data_ring.desc_size), 1849 rq->data_ring.base, 1850 rq->data_ring.basePA); 1851 rq->data_ring.base = NULL; 1852 rq->data_ring.desc_size = 0; 1853 } 1854 } 1855 } 1856 1857 static int 1858 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 1859 struct vmxnet3_adapter *adapter) 1860 { 1861 int i; 1862 1863 /* initialize buf_info */ 1864 for (i = 0; i < rq->rx_ring[0].size; i++) { 1865 1866 /* 1st buf for a pkt is skbuff */ 1867 if (i % adapter->rx_buf_per_pkt == 0) { 1868 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB; 1869 rq->buf_info[0][i].len = adapter->skb_buf_size; 1870 } else { /* subsequent bufs for a pkt is frag */ 1871 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 1872 rq->buf_info[0][i].len = PAGE_SIZE; 1873 } 1874 } 1875 for (i = 0; i < rq->rx_ring[1].size; i++) { 1876 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 1877 rq->buf_info[1][i].len = PAGE_SIZE; 1878 } 1879 1880 /* reset internal state and allocate buffers for both rings */ 1881 for (i = 0; i < 2; i++) { 1882 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 1883 1884 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 1885 sizeof(struct Vmxnet3_RxDesc)); 1886 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 1887 rq->rx_ring[i].isOutOfOrder = 0; 1888 } 1889 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 1890 adapter) == 0) { 1891 /* at least has 1 rx buffer for the 1st ring */ 1892 return -ENOMEM; 1893 } 1894 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 1895 1896 /* reset the comp ring */ 1897 rq->comp_ring.next2proc = 0; 1898 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 1899 sizeof(struct Vmxnet3_RxCompDesc)); 1900 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1901 1902 /* reset rxctx */ 1903 rq->rx_ctx.skb = NULL; 1904 1905 /* stats are not reset */ 1906 return 0; 1907 } 1908 1909 1910 static int 1911 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 1912 { 1913 int i, err = 0; 1914 1915 for (i = 0; i < adapter->num_rx_queues; i++) { 1916 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 1917 if (unlikely(err)) { 1918 dev_err(&adapter->netdev->dev, "%s: failed to " 1919 "initialize rx queue%i\n", 1920 adapter->netdev->name, i); 1921 break; 1922 } 1923 } 1924 return err; 1925 1926 } 1927 1928 1929 static int 1930 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1931 { 1932 int i; 1933 size_t sz; 1934 struct vmxnet3_rx_buf_info *bi; 1935 1936 for (i = 0; i < 2; i++) { 1937 1938 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 1939 rq->rx_ring[i].base = dma_alloc_coherent( 1940 &adapter->pdev->dev, sz, 1941 &rq->rx_ring[i].basePA, 1942 GFP_KERNEL); 1943 if (!rq->rx_ring[i].base) { 1944 netdev_err(adapter->netdev, 1945 "failed to allocate rx ring %d\n", i); 1946 goto err; 1947 } 1948 } 1949 1950 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 1951 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 1952 rq->data_ring.base = 1953 dma_alloc_coherent(&adapter->pdev->dev, sz, 1954 &rq->data_ring.basePA, 1955 GFP_KERNEL); 1956 if (!rq->data_ring.base) { 1957 netdev_err(adapter->netdev, 1958 "rx data ring will be disabled\n"); 1959 adapter->rxdataring_enabled = false; 1960 } 1961 } else { 1962 rq->data_ring.base = NULL; 1963 rq->data_ring.desc_size = 0; 1964 } 1965 1966 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 1967 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 1968 &rq->comp_ring.basePA, 1969 GFP_KERNEL); 1970 if (!rq->comp_ring.base) { 1971 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 1972 goto err; 1973 } 1974 1975 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 1976 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 1977 dev_to_node(&adapter->pdev->dev)); 1978 if (!bi) 1979 goto err; 1980 1981 rq->buf_info[0] = bi; 1982 rq->buf_info[1] = bi + rq->rx_ring[0].size; 1983 1984 return 0; 1985 1986 err: 1987 vmxnet3_rq_destroy(rq, adapter); 1988 return -ENOMEM; 1989 } 1990 1991 1992 static int 1993 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 1994 { 1995 int i, err = 0; 1996 1997 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 1998 1999 for (i = 0; i < adapter->num_rx_queues; i++) { 2000 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2001 if (unlikely(err)) { 2002 dev_err(&adapter->netdev->dev, 2003 "%s: failed to create rx queue%i\n", 2004 adapter->netdev->name, i); 2005 goto err_out; 2006 } 2007 } 2008 2009 if (!adapter->rxdataring_enabled) 2010 vmxnet3_rq_destroy_all_rxdataring(adapter); 2011 2012 return err; 2013 err_out: 2014 vmxnet3_rq_destroy_all(adapter); 2015 return err; 2016 2017 } 2018 2019 /* Multiple queue aware polling function for tx and rx */ 2020 2021 static int 2022 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2023 { 2024 int rcd_done = 0, i; 2025 if (unlikely(adapter->shared->ecr)) 2026 vmxnet3_process_events(adapter); 2027 for (i = 0; i < adapter->num_tx_queues; i++) 2028 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2029 2030 for (i = 0; i < adapter->num_rx_queues; i++) 2031 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2032 adapter, budget); 2033 return rcd_done; 2034 } 2035 2036 2037 static int 2038 vmxnet3_poll(struct napi_struct *napi, int budget) 2039 { 2040 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2041 struct vmxnet3_rx_queue, napi); 2042 int rxd_done; 2043 2044 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2045 2046 if (rxd_done < budget) { 2047 napi_complete_done(napi, rxd_done); 2048 vmxnet3_enable_all_intrs(rx_queue->adapter); 2049 } 2050 return rxd_done; 2051 } 2052 2053 /* 2054 * NAPI polling function for MSI-X mode with multiple Rx queues 2055 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2056 */ 2057 2058 static int 2059 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2060 { 2061 struct vmxnet3_rx_queue *rq = container_of(napi, 2062 struct vmxnet3_rx_queue, napi); 2063 struct vmxnet3_adapter *adapter = rq->adapter; 2064 int rxd_done; 2065 2066 /* When sharing interrupt with corresponding tx queue, process 2067 * tx completions in that queue as well 2068 */ 2069 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2070 struct vmxnet3_tx_queue *tq = 2071 &adapter->tx_queue[rq - adapter->rx_queue]; 2072 vmxnet3_tq_tx_complete(tq, adapter); 2073 } 2074 2075 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2076 2077 if (rxd_done < budget) { 2078 napi_complete_done(napi, rxd_done); 2079 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2080 } 2081 return rxd_done; 2082 } 2083 2084 2085 #ifdef CONFIG_PCI_MSI 2086 2087 /* 2088 * Handle completion interrupts on tx queues 2089 * Returns whether or not the intr is handled 2090 */ 2091 2092 static irqreturn_t 2093 vmxnet3_msix_tx(int irq, void *data) 2094 { 2095 struct vmxnet3_tx_queue *tq = data; 2096 struct vmxnet3_adapter *adapter = tq->adapter; 2097 2098 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2099 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2100 2101 /* Handle the case where only one irq is allocate for all tx queues */ 2102 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2103 int i; 2104 for (i = 0; i < adapter->num_tx_queues; i++) { 2105 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2106 vmxnet3_tq_tx_complete(txq, adapter); 2107 } 2108 } else { 2109 vmxnet3_tq_tx_complete(tq, adapter); 2110 } 2111 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2112 2113 return IRQ_HANDLED; 2114 } 2115 2116 2117 /* 2118 * Handle completion interrupts on rx queues. Returns whether or not the 2119 * intr is handled 2120 */ 2121 2122 static irqreturn_t 2123 vmxnet3_msix_rx(int irq, void *data) 2124 { 2125 struct vmxnet3_rx_queue *rq = data; 2126 struct vmxnet3_adapter *adapter = rq->adapter; 2127 2128 /* disable intr if needed */ 2129 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2130 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2131 napi_schedule(&rq->napi); 2132 2133 return IRQ_HANDLED; 2134 } 2135 2136 /* 2137 *---------------------------------------------------------------------------- 2138 * 2139 * vmxnet3_msix_event -- 2140 * 2141 * vmxnet3 msix event intr handler 2142 * 2143 * Result: 2144 * whether or not the intr is handled 2145 * 2146 *---------------------------------------------------------------------------- 2147 */ 2148 2149 static irqreturn_t 2150 vmxnet3_msix_event(int irq, void *data) 2151 { 2152 struct net_device *dev = data; 2153 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2154 2155 /* disable intr if needed */ 2156 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2157 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2158 2159 if (adapter->shared->ecr) 2160 vmxnet3_process_events(adapter); 2161 2162 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2163 2164 return IRQ_HANDLED; 2165 } 2166 2167 #endif /* CONFIG_PCI_MSI */ 2168 2169 2170 /* Interrupt handler for vmxnet3 */ 2171 static irqreturn_t 2172 vmxnet3_intr(int irq, void *dev_id) 2173 { 2174 struct net_device *dev = dev_id; 2175 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2176 2177 if (adapter->intr.type == VMXNET3_IT_INTX) { 2178 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2179 if (unlikely(icr == 0)) 2180 /* not ours */ 2181 return IRQ_NONE; 2182 } 2183 2184 2185 /* disable intr if needed */ 2186 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2187 vmxnet3_disable_all_intrs(adapter); 2188 2189 napi_schedule(&adapter->rx_queue[0].napi); 2190 2191 return IRQ_HANDLED; 2192 } 2193 2194 #ifdef CONFIG_NET_POLL_CONTROLLER 2195 2196 /* netpoll callback. */ 2197 static void 2198 vmxnet3_netpoll(struct net_device *netdev) 2199 { 2200 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2201 2202 switch (adapter->intr.type) { 2203 #ifdef CONFIG_PCI_MSI 2204 case VMXNET3_IT_MSIX: { 2205 int i; 2206 for (i = 0; i < adapter->num_rx_queues; i++) 2207 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2208 break; 2209 } 2210 #endif 2211 case VMXNET3_IT_MSI: 2212 default: 2213 vmxnet3_intr(0, adapter->netdev); 2214 break; 2215 } 2216 2217 } 2218 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2219 2220 static int 2221 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2222 { 2223 struct vmxnet3_intr *intr = &adapter->intr; 2224 int err = 0, i; 2225 int vector = 0; 2226 2227 #ifdef CONFIG_PCI_MSI 2228 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2229 for (i = 0; i < adapter->num_tx_queues; i++) { 2230 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2231 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2232 adapter->netdev->name, vector); 2233 err = request_irq( 2234 intr->msix_entries[vector].vector, 2235 vmxnet3_msix_tx, 0, 2236 adapter->tx_queue[i].name, 2237 &adapter->tx_queue[i]); 2238 } else { 2239 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2240 adapter->netdev->name, vector); 2241 } 2242 if (err) { 2243 dev_err(&adapter->netdev->dev, 2244 "Failed to request irq for MSIX, %s, " 2245 "error %d\n", 2246 adapter->tx_queue[i].name, err); 2247 return err; 2248 } 2249 2250 /* Handle the case where only 1 MSIx was allocated for 2251 * all tx queues */ 2252 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2253 for (; i < adapter->num_tx_queues; i++) 2254 adapter->tx_queue[i].comp_ring.intr_idx 2255 = vector; 2256 vector++; 2257 break; 2258 } else { 2259 adapter->tx_queue[i].comp_ring.intr_idx 2260 = vector++; 2261 } 2262 } 2263 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2264 vector = 0; 2265 2266 for (i = 0; i < adapter->num_rx_queues; i++) { 2267 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2268 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2269 adapter->netdev->name, vector); 2270 else 2271 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2272 adapter->netdev->name, vector); 2273 err = request_irq(intr->msix_entries[vector].vector, 2274 vmxnet3_msix_rx, 0, 2275 adapter->rx_queue[i].name, 2276 &(adapter->rx_queue[i])); 2277 if (err) { 2278 netdev_err(adapter->netdev, 2279 "Failed to request irq for MSIX, " 2280 "%s, error %d\n", 2281 adapter->rx_queue[i].name, err); 2282 return err; 2283 } 2284 2285 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2286 } 2287 2288 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2289 adapter->netdev->name, vector); 2290 err = request_irq(intr->msix_entries[vector].vector, 2291 vmxnet3_msix_event, 0, 2292 intr->event_msi_vector_name, adapter->netdev); 2293 intr->event_intr_idx = vector; 2294 2295 } else if (intr->type == VMXNET3_IT_MSI) { 2296 adapter->num_rx_queues = 1; 2297 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2298 adapter->netdev->name, adapter->netdev); 2299 } else { 2300 #endif 2301 adapter->num_rx_queues = 1; 2302 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2303 IRQF_SHARED, adapter->netdev->name, 2304 adapter->netdev); 2305 #ifdef CONFIG_PCI_MSI 2306 } 2307 #endif 2308 intr->num_intrs = vector + 1; 2309 if (err) { 2310 netdev_err(adapter->netdev, 2311 "Failed to request irq (intr type:%d), error %d\n", 2312 intr->type, err); 2313 } else { 2314 /* Number of rx queues will not change after this */ 2315 for (i = 0; i < adapter->num_rx_queues; i++) { 2316 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2317 rq->qid = i; 2318 rq->qid2 = i + adapter->num_rx_queues; 2319 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2320 } 2321 2322 /* init our intr settings */ 2323 for (i = 0; i < intr->num_intrs; i++) 2324 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2325 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2326 adapter->intr.event_intr_idx = 0; 2327 for (i = 0; i < adapter->num_tx_queues; i++) 2328 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2329 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2330 } 2331 2332 netdev_info(adapter->netdev, 2333 "intr type %u, mode %u, %u vectors allocated\n", 2334 intr->type, intr->mask_mode, intr->num_intrs); 2335 } 2336 2337 return err; 2338 } 2339 2340 2341 static void 2342 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2343 { 2344 struct vmxnet3_intr *intr = &adapter->intr; 2345 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2346 2347 switch (intr->type) { 2348 #ifdef CONFIG_PCI_MSI 2349 case VMXNET3_IT_MSIX: 2350 { 2351 int i, vector = 0; 2352 2353 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2354 for (i = 0; i < adapter->num_tx_queues; i++) { 2355 free_irq(intr->msix_entries[vector++].vector, 2356 &(adapter->tx_queue[i])); 2357 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2358 break; 2359 } 2360 } 2361 2362 for (i = 0; i < adapter->num_rx_queues; i++) { 2363 free_irq(intr->msix_entries[vector++].vector, 2364 &(adapter->rx_queue[i])); 2365 } 2366 2367 free_irq(intr->msix_entries[vector].vector, 2368 adapter->netdev); 2369 BUG_ON(vector >= intr->num_intrs); 2370 break; 2371 } 2372 #endif 2373 case VMXNET3_IT_MSI: 2374 free_irq(adapter->pdev->irq, adapter->netdev); 2375 break; 2376 case VMXNET3_IT_INTX: 2377 free_irq(adapter->pdev->irq, adapter->netdev); 2378 break; 2379 default: 2380 BUG(); 2381 } 2382 } 2383 2384 2385 static void 2386 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2387 { 2388 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2389 u16 vid; 2390 2391 /* allow untagged pkts */ 2392 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2393 2394 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2395 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2396 } 2397 2398 2399 static int 2400 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2401 { 2402 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2403 2404 if (!(netdev->flags & IFF_PROMISC)) { 2405 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2406 unsigned long flags; 2407 2408 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2409 spin_lock_irqsave(&adapter->cmd_lock, flags); 2410 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2411 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2412 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2413 } 2414 2415 set_bit(vid, adapter->active_vlans); 2416 2417 return 0; 2418 } 2419 2420 2421 static int 2422 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2423 { 2424 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2425 2426 if (!(netdev->flags & IFF_PROMISC)) { 2427 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2428 unsigned long flags; 2429 2430 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2431 spin_lock_irqsave(&adapter->cmd_lock, flags); 2432 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2433 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2434 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2435 } 2436 2437 clear_bit(vid, adapter->active_vlans); 2438 2439 return 0; 2440 } 2441 2442 2443 static u8 * 2444 vmxnet3_copy_mc(struct net_device *netdev) 2445 { 2446 u8 *buf = NULL; 2447 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2448 2449 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2450 if (sz <= 0xffff) { 2451 /* We may be called with BH disabled */ 2452 buf = kmalloc(sz, GFP_ATOMIC); 2453 if (buf) { 2454 struct netdev_hw_addr *ha; 2455 int i = 0; 2456 2457 netdev_for_each_mc_addr(ha, netdev) 2458 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2459 ETH_ALEN); 2460 } 2461 } 2462 return buf; 2463 } 2464 2465 2466 static void 2467 vmxnet3_set_mc(struct net_device *netdev) 2468 { 2469 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2470 unsigned long flags; 2471 struct Vmxnet3_RxFilterConf *rxConf = 2472 &adapter->shared->devRead.rxFilterConf; 2473 u8 *new_table = NULL; 2474 dma_addr_t new_table_pa = 0; 2475 bool new_table_pa_valid = false; 2476 u32 new_mode = VMXNET3_RXM_UCAST; 2477 2478 if (netdev->flags & IFF_PROMISC) { 2479 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2480 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2481 2482 new_mode |= VMXNET3_RXM_PROMISC; 2483 } else { 2484 vmxnet3_restore_vlan(adapter); 2485 } 2486 2487 if (netdev->flags & IFF_BROADCAST) 2488 new_mode |= VMXNET3_RXM_BCAST; 2489 2490 if (netdev->flags & IFF_ALLMULTI) 2491 new_mode |= VMXNET3_RXM_ALL_MULTI; 2492 else 2493 if (!netdev_mc_empty(netdev)) { 2494 new_table = vmxnet3_copy_mc(netdev); 2495 if (new_table) { 2496 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2497 2498 rxConf->mfTableLen = cpu_to_le16(sz); 2499 new_table_pa = dma_map_single( 2500 &adapter->pdev->dev, 2501 new_table, 2502 sz, 2503 DMA_TO_DEVICE); 2504 if (!dma_mapping_error(&adapter->pdev->dev, 2505 new_table_pa)) { 2506 new_mode |= VMXNET3_RXM_MCAST; 2507 new_table_pa_valid = true; 2508 rxConf->mfTablePA = cpu_to_le64( 2509 new_table_pa); 2510 } 2511 } 2512 if (!new_table_pa_valid) { 2513 netdev_info(netdev, 2514 "failed to copy mcast list, setting ALL_MULTI\n"); 2515 new_mode |= VMXNET3_RXM_ALL_MULTI; 2516 } 2517 } 2518 2519 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2520 rxConf->mfTableLen = 0; 2521 rxConf->mfTablePA = 0; 2522 } 2523 2524 spin_lock_irqsave(&adapter->cmd_lock, flags); 2525 if (new_mode != rxConf->rxMode) { 2526 rxConf->rxMode = cpu_to_le32(new_mode); 2527 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2528 VMXNET3_CMD_UPDATE_RX_MODE); 2529 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2530 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2531 } 2532 2533 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2534 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2535 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2536 2537 if (new_table_pa_valid) 2538 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2539 rxConf->mfTableLen, DMA_TO_DEVICE); 2540 kfree(new_table); 2541 } 2542 2543 void 2544 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2545 { 2546 int i; 2547 2548 for (i = 0; i < adapter->num_rx_queues; i++) 2549 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2550 } 2551 2552 2553 /* 2554 * Set up driver_shared based on settings in adapter. 2555 */ 2556 2557 static void 2558 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2559 { 2560 struct Vmxnet3_DriverShared *shared = adapter->shared; 2561 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2562 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2563 struct Vmxnet3_TxQueueConf *tqc; 2564 struct Vmxnet3_RxQueueConf *rqc; 2565 int i; 2566 2567 memset(shared, 0, sizeof(*shared)); 2568 2569 /* driver settings */ 2570 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2571 devRead->misc.driverInfo.version = cpu_to_le32( 2572 VMXNET3_DRIVER_VERSION_NUM); 2573 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2574 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2575 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2576 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2577 *((u32 *)&devRead->misc.driverInfo.gos)); 2578 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2579 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2580 2581 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2582 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2583 2584 /* set up feature flags */ 2585 if (adapter->netdev->features & NETIF_F_RXCSUM) 2586 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2587 2588 if (adapter->netdev->features & NETIF_F_LRO) { 2589 devRead->misc.uptFeatures |= UPT1_F_LRO; 2590 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2591 } 2592 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2593 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2594 2595 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2596 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2597 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2598 2599 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2600 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2601 devRead->misc.queueDescLen = cpu_to_le32( 2602 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2603 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2604 2605 /* tx queue settings */ 2606 devRead->misc.numTxQueues = adapter->num_tx_queues; 2607 for (i = 0; i < adapter->num_tx_queues; i++) { 2608 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2609 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2610 tqc = &adapter->tqd_start[i].conf; 2611 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2612 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2613 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2614 tqc->ddPA = cpu_to_le64(~0ULL); 2615 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2616 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2617 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2618 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2619 tqc->ddLen = cpu_to_le32(0); 2620 tqc->intrIdx = tq->comp_ring.intr_idx; 2621 } 2622 2623 /* rx queue settings */ 2624 devRead->misc.numRxQueues = adapter->num_rx_queues; 2625 for (i = 0; i < adapter->num_rx_queues; i++) { 2626 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2627 rqc = &adapter->rqd_start[i].conf; 2628 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2629 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2630 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2631 rqc->ddPA = cpu_to_le64(~0ULL); 2632 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2633 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2634 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2635 rqc->ddLen = cpu_to_le32(0); 2636 rqc->intrIdx = rq->comp_ring.intr_idx; 2637 if (VMXNET3_VERSION_GE_3(adapter)) { 2638 rqc->rxDataRingBasePA = 2639 cpu_to_le64(rq->data_ring.basePA); 2640 rqc->rxDataRingDescSize = 2641 cpu_to_le16(rq->data_ring.desc_size); 2642 } 2643 } 2644 2645 #ifdef VMXNET3_RSS 2646 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2647 2648 if (adapter->rss) { 2649 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2650 2651 devRead->misc.uptFeatures |= UPT1_F_RSS; 2652 devRead->misc.numRxQueues = adapter->num_rx_queues; 2653 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2654 UPT1_RSS_HASH_TYPE_IPV4 | 2655 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2656 UPT1_RSS_HASH_TYPE_IPV6; 2657 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2658 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2659 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2660 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2661 2662 for (i = 0; i < rssConf->indTableSize; i++) 2663 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2664 i, adapter->num_rx_queues); 2665 2666 devRead->rssConfDesc.confVer = 1; 2667 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2668 devRead->rssConfDesc.confPA = 2669 cpu_to_le64(adapter->rss_conf_pa); 2670 } 2671 2672 #endif /* VMXNET3_RSS */ 2673 2674 /* intr settings */ 2675 if (!VMXNET3_VERSION_GE_6(adapter) || 2676 !adapter->queuesExtEnabled) { 2677 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2678 VMXNET3_IMM_AUTO; 2679 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2680 for (i = 0; i < adapter->intr.num_intrs; i++) 2681 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2682 2683 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2684 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2685 } else { 2686 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 2687 VMXNET3_IMM_AUTO; 2688 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 2689 for (i = 0; i < adapter->intr.num_intrs; i++) 2690 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 2691 2692 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 2693 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2694 } 2695 2696 /* rx filter settings */ 2697 devRead->rxFilterConf.rxMode = 0; 2698 vmxnet3_restore_vlan(adapter); 2699 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 2700 2701 /* the rest are already zeroed */ 2702 } 2703 2704 static void 2705 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 2706 { 2707 struct Vmxnet3_DriverShared *shared = adapter->shared; 2708 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2709 unsigned long flags; 2710 2711 if (!VMXNET3_VERSION_GE_7(adapter)) 2712 return; 2713 2714 cmdInfo->ringBufSize = adapter->ringBufSize; 2715 spin_lock_irqsave(&adapter->cmd_lock, flags); 2716 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2717 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 2718 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2719 } 2720 2721 static void 2722 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 2723 { 2724 struct Vmxnet3_DriverShared *shared = adapter->shared; 2725 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2726 unsigned long flags; 2727 2728 if (!VMXNET3_VERSION_GE_3(adapter)) 2729 return; 2730 2731 spin_lock_irqsave(&adapter->cmd_lock, flags); 2732 cmdInfo->varConf.confVer = 1; 2733 cmdInfo->varConf.confLen = 2734 cpu_to_le32(sizeof(*adapter->coal_conf)); 2735 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 2736 2737 if (adapter->default_coal_mode) { 2738 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2739 VMXNET3_CMD_GET_COALESCE); 2740 } else { 2741 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2742 VMXNET3_CMD_SET_COALESCE); 2743 } 2744 2745 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2746 } 2747 2748 static void 2749 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 2750 { 2751 struct Vmxnet3_DriverShared *shared = adapter->shared; 2752 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2753 unsigned long flags; 2754 2755 if (!VMXNET3_VERSION_GE_4(adapter)) 2756 return; 2757 2758 spin_lock_irqsave(&adapter->cmd_lock, flags); 2759 2760 if (adapter->default_rss_fields) { 2761 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2762 VMXNET3_CMD_GET_RSS_FIELDS); 2763 adapter->rss_fields = 2764 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2765 } else { 2766 if (VMXNET3_VERSION_GE_7(adapter)) { 2767 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 2768 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 2769 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2770 VMXNET3_CAP_UDP_RSS)) { 2771 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 2772 } else { 2773 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 2774 } 2775 2776 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 2777 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2778 VMXNET3_CAP_ESP_RSS_IPV4)) { 2779 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 2780 } else { 2781 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 2782 } 2783 2784 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 2785 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2786 VMXNET3_CAP_ESP_RSS_IPV6)) { 2787 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 2788 } else { 2789 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 2790 } 2791 2792 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 2793 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 2794 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2795 } 2796 cmdInfo->setRssFields = adapter->rss_fields; 2797 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2798 VMXNET3_CMD_SET_RSS_FIELDS); 2799 /* Not all requested RSS may get applied, so get and 2800 * cache what was actually applied. 2801 */ 2802 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2803 VMXNET3_CMD_GET_RSS_FIELDS); 2804 adapter->rss_fields = 2805 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2806 } 2807 2808 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2809 } 2810 2811 int 2812 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2813 { 2814 int err, i; 2815 u32 ret; 2816 unsigned long flags; 2817 2818 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 2819 " ring sizes %u %u %u\n", adapter->netdev->name, 2820 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 2821 adapter->tx_queue[0].tx_ring.size, 2822 adapter->rx_queue[0].rx_ring[0].size, 2823 adapter->rx_queue[0].rx_ring[1].size); 2824 2825 vmxnet3_tq_init_all(adapter); 2826 err = vmxnet3_rq_init_all(adapter); 2827 if (err) { 2828 netdev_err(adapter->netdev, 2829 "Failed to init rx queue error %d\n", err); 2830 goto rq_err; 2831 } 2832 2833 err = vmxnet3_request_irqs(adapter); 2834 if (err) { 2835 netdev_err(adapter->netdev, 2836 "Failed to setup irq for error %d\n", err); 2837 goto irq_err; 2838 } 2839 2840 vmxnet3_setup_driver_shared(adapter); 2841 2842 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 2843 adapter->shared_pa)); 2844 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 2845 adapter->shared_pa)); 2846 spin_lock_irqsave(&adapter->cmd_lock, flags); 2847 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2848 VMXNET3_CMD_ACTIVATE_DEV); 2849 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2850 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2851 2852 if (ret != 0) { 2853 netdev_err(adapter->netdev, 2854 "Failed to activate dev: error %u\n", ret); 2855 err = -EINVAL; 2856 goto activate_err; 2857 } 2858 2859 vmxnet3_init_bufsize(adapter); 2860 vmxnet3_init_coalesce(adapter); 2861 vmxnet3_init_rssfields(adapter); 2862 2863 for (i = 0; i < adapter->num_rx_queues; i++) { 2864 VMXNET3_WRITE_BAR0_REG(adapter, 2865 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 2866 adapter->rx_queue[i].rx_ring[0].next2fill); 2867 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 2868 (i * VMXNET3_REG_ALIGN)), 2869 adapter->rx_queue[i].rx_ring[1].next2fill); 2870 } 2871 2872 /* Apply the rx filter settins last. */ 2873 vmxnet3_set_mc(adapter->netdev); 2874 2875 /* 2876 * Check link state when first activating device. It will start the 2877 * tx queue if the link is up. 2878 */ 2879 vmxnet3_check_link(adapter, true); 2880 netif_tx_wake_all_queues(adapter->netdev); 2881 for (i = 0; i < adapter->num_rx_queues; i++) 2882 napi_enable(&adapter->rx_queue[i].napi); 2883 vmxnet3_enable_all_intrs(adapter); 2884 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2885 return 0; 2886 2887 activate_err: 2888 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 2889 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 2890 vmxnet3_free_irqs(adapter); 2891 irq_err: 2892 rq_err: 2893 /* free up buffers we allocated */ 2894 vmxnet3_rq_cleanup_all(adapter); 2895 return err; 2896 } 2897 2898 2899 void 2900 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 2901 { 2902 unsigned long flags; 2903 spin_lock_irqsave(&adapter->cmd_lock, flags); 2904 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 2905 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2906 } 2907 2908 2909 int 2910 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2911 { 2912 int i; 2913 unsigned long flags; 2914 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2915 return 0; 2916 2917 2918 spin_lock_irqsave(&adapter->cmd_lock, flags); 2919 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2920 VMXNET3_CMD_QUIESCE_DEV); 2921 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2922 vmxnet3_disable_all_intrs(adapter); 2923 2924 for (i = 0; i < adapter->num_rx_queues; i++) 2925 napi_disable(&adapter->rx_queue[i].napi); 2926 netif_tx_disable(adapter->netdev); 2927 adapter->link_speed = 0; 2928 netif_carrier_off(adapter->netdev); 2929 2930 vmxnet3_tq_cleanup_all(adapter); 2931 vmxnet3_rq_cleanup_all(adapter); 2932 vmxnet3_free_irqs(adapter); 2933 return 0; 2934 } 2935 2936 2937 static void 2938 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 2939 { 2940 u32 tmp; 2941 2942 tmp = *(u32 *)mac; 2943 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 2944 2945 tmp = (mac[5] << 8) | mac[4]; 2946 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 2947 } 2948 2949 2950 static int 2951 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 2952 { 2953 struct sockaddr *addr = p; 2954 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2955 2956 dev_addr_set(netdev, addr->sa_data); 2957 vmxnet3_write_mac_addr(adapter, addr->sa_data); 2958 2959 return 0; 2960 } 2961 2962 2963 /* ==================== initialization and cleanup routines ============ */ 2964 2965 static int 2966 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 2967 { 2968 int err; 2969 unsigned long mmio_start, mmio_len; 2970 struct pci_dev *pdev = adapter->pdev; 2971 2972 err = pci_enable_device(pdev); 2973 if (err) { 2974 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 2975 return err; 2976 } 2977 2978 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 2979 vmxnet3_driver_name); 2980 if (err) { 2981 dev_err(&pdev->dev, 2982 "Failed to request region for adapter: error %d\n", err); 2983 goto err_enable_device; 2984 } 2985 2986 pci_set_master(pdev); 2987 2988 mmio_start = pci_resource_start(pdev, 0); 2989 mmio_len = pci_resource_len(pdev, 0); 2990 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 2991 if (!adapter->hw_addr0) { 2992 dev_err(&pdev->dev, "Failed to map bar0\n"); 2993 err = -EIO; 2994 goto err_ioremap; 2995 } 2996 2997 mmio_start = pci_resource_start(pdev, 1); 2998 mmio_len = pci_resource_len(pdev, 1); 2999 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3000 if (!adapter->hw_addr1) { 3001 dev_err(&pdev->dev, "Failed to map bar1\n"); 3002 err = -EIO; 3003 goto err_bar1; 3004 } 3005 return 0; 3006 3007 err_bar1: 3008 iounmap(adapter->hw_addr0); 3009 err_ioremap: 3010 pci_release_selected_regions(pdev, (1 << 2) - 1); 3011 err_enable_device: 3012 pci_disable_device(pdev); 3013 return err; 3014 } 3015 3016 3017 static void 3018 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3019 { 3020 BUG_ON(!adapter->pdev); 3021 3022 iounmap(adapter->hw_addr0); 3023 iounmap(adapter->hw_addr1); 3024 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3025 pci_disable_device(adapter->pdev); 3026 } 3027 3028 3029 static void 3030 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3031 { 3032 size_t sz, i, ring0_size, ring1_size, comp_size; 3033 /* With version7 ring1 will have only T0 buffers */ 3034 if (!VMXNET3_VERSION_GE_7(adapter)) { 3035 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3036 VMXNET3_MAX_ETH_HDR_SIZE) { 3037 adapter->skb_buf_size = adapter->netdev->mtu + 3038 VMXNET3_MAX_ETH_HDR_SIZE; 3039 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3040 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3041 3042 adapter->rx_buf_per_pkt = 1; 3043 } else { 3044 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3045 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3046 VMXNET3_MAX_ETH_HDR_SIZE; 3047 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3048 } 3049 } else { 3050 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3051 VMXNET3_MAX_SKB_BUF_SIZE); 3052 adapter->rx_buf_per_pkt = 1; 3053 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3054 adapter->ringBufSize.ring1BufSizeType1 = 0; 3055 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3056 } 3057 3058 /* 3059 * for simplicity, force the ring0 size to be a multiple of 3060 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3061 */ 3062 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3063 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3064 ring0_size = (ring0_size + sz - 1) / sz * sz; 3065 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3066 sz * sz); 3067 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3068 ring1_size = (ring1_size + sz - 1) / sz * sz; 3069 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3070 sz * sz); 3071 /* For v7 and later, keep ring size power of 2 for UPT */ 3072 if (VMXNET3_VERSION_GE_7(adapter)) { 3073 ring0_size = rounddown_pow_of_two(ring0_size); 3074 ring1_size = rounddown_pow_of_two(ring1_size); 3075 } 3076 comp_size = ring0_size + ring1_size; 3077 3078 for (i = 0; i < adapter->num_rx_queues; i++) { 3079 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3080 3081 rq->rx_ring[0].size = ring0_size; 3082 rq->rx_ring[1].size = ring1_size; 3083 rq->comp_ring.size = comp_size; 3084 } 3085 } 3086 3087 3088 int 3089 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3090 u32 rx_ring_size, u32 rx_ring2_size, 3091 u16 txdata_desc_size, u16 rxdata_desc_size) 3092 { 3093 int err = 0, i; 3094 3095 for (i = 0; i < adapter->num_tx_queues; i++) { 3096 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3097 tq->tx_ring.size = tx_ring_size; 3098 tq->data_ring.size = tx_ring_size; 3099 tq->comp_ring.size = tx_ring_size; 3100 tq->txdata_desc_size = txdata_desc_size; 3101 tq->shared = &adapter->tqd_start[i].ctrl; 3102 tq->stopped = true; 3103 tq->adapter = adapter; 3104 tq->qid = i; 3105 err = vmxnet3_tq_create(tq, adapter); 3106 /* 3107 * Too late to change num_tx_queues. We cannot do away with 3108 * lesser number of queues than what we asked for 3109 */ 3110 if (err) 3111 goto queue_err; 3112 } 3113 3114 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3115 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3116 vmxnet3_adjust_rx_ring_size(adapter); 3117 3118 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3119 for (i = 0; i < adapter->num_rx_queues; i++) { 3120 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3121 /* qid and qid2 for rx queues will be assigned later when num 3122 * of rx queues is finalized after allocating intrs */ 3123 rq->shared = &adapter->rqd_start[i].ctrl; 3124 rq->adapter = adapter; 3125 rq->data_ring.desc_size = rxdata_desc_size; 3126 err = vmxnet3_rq_create(rq, adapter); 3127 if (err) { 3128 if (i == 0) { 3129 netdev_err(adapter->netdev, 3130 "Could not allocate any rx queues. " 3131 "Aborting.\n"); 3132 goto queue_err; 3133 } else { 3134 netdev_info(adapter->netdev, 3135 "Number of rx queues changed " 3136 "to : %d.\n", i); 3137 adapter->num_rx_queues = i; 3138 err = 0; 3139 break; 3140 } 3141 } 3142 } 3143 3144 if (!adapter->rxdataring_enabled) 3145 vmxnet3_rq_destroy_all_rxdataring(adapter); 3146 3147 return err; 3148 queue_err: 3149 vmxnet3_tq_destroy_all(adapter); 3150 return err; 3151 } 3152 3153 static int 3154 vmxnet3_open(struct net_device *netdev) 3155 { 3156 struct vmxnet3_adapter *adapter; 3157 int err, i; 3158 3159 adapter = netdev_priv(netdev); 3160 3161 for (i = 0; i < adapter->num_tx_queues; i++) 3162 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3163 3164 if (VMXNET3_VERSION_GE_3(adapter)) { 3165 unsigned long flags; 3166 u16 txdata_desc_size; 3167 3168 spin_lock_irqsave(&adapter->cmd_lock, flags); 3169 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3170 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3171 txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter, 3172 VMXNET3_REG_CMD); 3173 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3174 3175 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3176 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3177 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3178 adapter->txdata_desc_size = 3179 sizeof(struct Vmxnet3_TxDataDesc); 3180 } else { 3181 adapter->txdata_desc_size = txdata_desc_size; 3182 } 3183 } else { 3184 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3185 } 3186 3187 err = vmxnet3_create_queues(adapter, 3188 adapter->tx_ring_size, 3189 adapter->rx_ring_size, 3190 adapter->rx_ring2_size, 3191 adapter->txdata_desc_size, 3192 adapter->rxdata_desc_size); 3193 if (err) 3194 goto queue_err; 3195 3196 err = vmxnet3_activate_dev(adapter); 3197 if (err) 3198 goto activate_err; 3199 3200 return 0; 3201 3202 activate_err: 3203 vmxnet3_rq_destroy_all(adapter); 3204 vmxnet3_tq_destroy_all(adapter); 3205 queue_err: 3206 return err; 3207 } 3208 3209 3210 static int 3211 vmxnet3_close(struct net_device *netdev) 3212 { 3213 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3214 3215 /* 3216 * Reset_work may be in the middle of resetting the device, wait for its 3217 * completion. 3218 */ 3219 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3220 usleep_range(1000, 2000); 3221 3222 vmxnet3_quiesce_dev(adapter); 3223 3224 vmxnet3_rq_destroy_all(adapter); 3225 vmxnet3_tq_destroy_all(adapter); 3226 3227 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3228 3229 3230 return 0; 3231 } 3232 3233 3234 void 3235 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3236 { 3237 int i; 3238 3239 /* 3240 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3241 * vmxnet3_close() will deadlock. 3242 */ 3243 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3244 3245 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3246 for (i = 0; i < adapter->num_rx_queues; i++) 3247 napi_enable(&adapter->rx_queue[i].napi); 3248 /* 3249 * Need to clear the quiesce bit to ensure that vmxnet3_close 3250 * can quiesce the device properly 3251 */ 3252 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3253 dev_close(adapter->netdev); 3254 } 3255 3256 3257 static int 3258 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3259 { 3260 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3261 int err = 0; 3262 3263 netdev->mtu = new_mtu; 3264 3265 /* 3266 * Reset_work may be in the middle of resetting the device, wait for its 3267 * completion. 3268 */ 3269 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3270 usleep_range(1000, 2000); 3271 3272 if (netif_running(netdev)) { 3273 vmxnet3_quiesce_dev(adapter); 3274 vmxnet3_reset_dev(adapter); 3275 3276 /* we need to re-create the rx queue based on the new mtu */ 3277 vmxnet3_rq_destroy_all(adapter); 3278 vmxnet3_adjust_rx_ring_size(adapter); 3279 err = vmxnet3_rq_create_all(adapter); 3280 if (err) { 3281 netdev_err(netdev, 3282 "failed to re-create rx queues, " 3283 " error %d. Closing it.\n", err); 3284 goto out; 3285 } 3286 3287 err = vmxnet3_activate_dev(adapter); 3288 if (err) { 3289 netdev_err(netdev, 3290 "failed to re-activate, error %d. " 3291 "Closing it\n", err); 3292 goto out; 3293 } 3294 } 3295 3296 out: 3297 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3298 if (err) 3299 vmxnet3_force_close(adapter); 3300 3301 return err; 3302 } 3303 3304 3305 static void 3306 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3307 { 3308 struct net_device *netdev = adapter->netdev; 3309 3310 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3311 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3312 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3313 NETIF_F_LRO | NETIF_F_HIGHDMA; 3314 3315 if (VMXNET3_VERSION_GE_4(adapter)) { 3316 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3317 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3318 3319 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3320 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3321 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3322 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3323 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3324 } 3325 3326 if (VMXNET3_VERSION_GE_7(adapter)) { 3327 unsigned long flags; 3328 3329 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3330 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3331 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3332 } 3333 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3334 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3335 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3336 } 3337 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3338 VMXNET3_CAP_GENEVE_TSO)) { 3339 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3340 } 3341 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3342 VMXNET3_CAP_VXLAN_TSO)) { 3343 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3344 } 3345 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3346 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3347 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3348 } 3349 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3350 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3351 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3352 } 3353 3354 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3355 spin_lock_irqsave(&adapter->cmd_lock, flags); 3356 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3357 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3358 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3359 3360 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3361 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3362 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3363 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3364 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3365 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3366 } 3367 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3368 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3369 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3370 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3371 } 3372 } 3373 3374 netdev->vlan_features = netdev->hw_features & 3375 ~(NETIF_F_HW_VLAN_CTAG_TX | 3376 NETIF_F_HW_VLAN_CTAG_RX); 3377 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3378 } 3379 3380 3381 static void 3382 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3383 { 3384 u32 tmp; 3385 3386 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3387 *(u32 *)mac = tmp; 3388 3389 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3390 mac[4] = tmp & 0xff; 3391 mac[5] = (tmp >> 8) & 0xff; 3392 } 3393 3394 #ifdef CONFIG_PCI_MSI 3395 3396 /* 3397 * Enable MSIx vectors. 3398 * Returns : 3399 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3400 * were enabled. 3401 * number of vectors which were enabled otherwise (this number is greater 3402 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3403 */ 3404 3405 static int 3406 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3407 { 3408 int ret = pci_enable_msix_range(adapter->pdev, 3409 adapter->intr.msix_entries, nvec, nvec); 3410 3411 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3412 dev_err(&adapter->netdev->dev, 3413 "Failed to enable %d MSI-X, trying %d\n", 3414 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3415 3416 ret = pci_enable_msix_range(adapter->pdev, 3417 adapter->intr.msix_entries, 3418 VMXNET3_LINUX_MIN_MSIX_VECT, 3419 VMXNET3_LINUX_MIN_MSIX_VECT); 3420 } 3421 3422 if (ret < 0) { 3423 dev_err(&adapter->netdev->dev, 3424 "Failed to enable MSI-X, error: %d\n", ret); 3425 } 3426 3427 return ret; 3428 } 3429 3430 3431 #endif /* CONFIG_PCI_MSI */ 3432 3433 static void 3434 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3435 { 3436 u32 cfg; 3437 unsigned long flags; 3438 3439 /* intr settings */ 3440 spin_lock_irqsave(&adapter->cmd_lock, flags); 3441 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3442 VMXNET3_CMD_GET_CONF_INTR); 3443 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3444 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3445 adapter->intr.type = cfg & 0x3; 3446 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3447 3448 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3449 adapter->intr.type = VMXNET3_IT_MSIX; 3450 } 3451 3452 #ifdef CONFIG_PCI_MSI 3453 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3454 int i, nvec, nvec_allocated; 3455 3456 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3457 1 : adapter->num_tx_queues; 3458 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3459 0 : adapter->num_rx_queues; 3460 nvec += 1; /* for link event */ 3461 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3462 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3463 3464 for (i = 0; i < nvec; i++) 3465 adapter->intr.msix_entries[i].entry = i; 3466 3467 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3468 if (nvec_allocated < 0) 3469 goto msix_err; 3470 3471 /* If we cannot allocate one MSIx vector per queue 3472 * then limit the number of rx queues to 1 3473 */ 3474 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3475 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3476 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3477 || adapter->num_rx_queues != 1) { 3478 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3479 netdev_err(adapter->netdev, 3480 "Number of rx queues : 1\n"); 3481 adapter->num_rx_queues = 1; 3482 } 3483 } 3484 3485 adapter->intr.num_intrs = nvec_allocated; 3486 return; 3487 3488 msix_err: 3489 /* If we cannot allocate MSIx vectors use only one rx queue */ 3490 dev_info(&adapter->pdev->dev, 3491 "Failed to enable MSI-X, error %d. " 3492 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3493 3494 adapter->intr.type = VMXNET3_IT_MSI; 3495 } 3496 3497 if (adapter->intr.type == VMXNET3_IT_MSI) { 3498 if (!pci_enable_msi(adapter->pdev)) { 3499 adapter->num_rx_queues = 1; 3500 adapter->intr.num_intrs = 1; 3501 return; 3502 } 3503 } 3504 #endif /* CONFIG_PCI_MSI */ 3505 3506 adapter->num_rx_queues = 1; 3507 dev_info(&adapter->netdev->dev, 3508 "Using INTx interrupt, #Rx queues: 1.\n"); 3509 adapter->intr.type = VMXNET3_IT_INTX; 3510 3511 /* INT-X related setting */ 3512 adapter->intr.num_intrs = 1; 3513 } 3514 3515 3516 static void 3517 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3518 { 3519 if (adapter->intr.type == VMXNET3_IT_MSIX) 3520 pci_disable_msix(adapter->pdev); 3521 else if (adapter->intr.type == VMXNET3_IT_MSI) 3522 pci_disable_msi(adapter->pdev); 3523 else 3524 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3525 } 3526 3527 3528 static void 3529 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3530 { 3531 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3532 adapter->tx_timeout_count++; 3533 3534 netdev_err(adapter->netdev, "tx hang\n"); 3535 schedule_work(&adapter->work); 3536 } 3537 3538 3539 static void 3540 vmxnet3_reset_work(struct work_struct *data) 3541 { 3542 struct vmxnet3_adapter *adapter; 3543 3544 adapter = container_of(data, struct vmxnet3_adapter, work); 3545 3546 /* if another thread is resetting the device, no need to proceed */ 3547 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3548 return; 3549 3550 /* if the device is closed, we must leave it alone */ 3551 rtnl_lock(); 3552 if (netif_running(adapter->netdev)) { 3553 netdev_notice(adapter->netdev, "resetting\n"); 3554 vmxnet3_quiesce_dev(adapter); 3555 vmxnet3_reset_dev(adapter); 3556 vmxnet3_activate_dev(adapter); 3557 } else { 3558 netdev_info(adapter->netdev, "already closed\n"); 3559 } 3560 rtnl_unlock(); 3561 3562 netif_wake_queue(adapter->netdev); 3563 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3564 } 3565 3566 3567 static int 3568 vmxnet3_probe_device(struct pci_dev *pdev, 3569 const struct pci_device_id *id) 3570 { 3571 static const struct net_device_ops vmxnet3_netdev_ops = { 3572 .ndo_open = vmxnet3_open, 3573 .ndo_stop = vmxnet3_close, 3574 .ndo_start_xmit = vmxnet3_xmit_frame, 3575 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3576 .ndo_change_mtu = vmxnet3_change_mtu, 3577 .ndo_fix_features = vmxnet3_fix_features, 3578 .ndo_set_features = vmxnet3_set_features, 3579 .ndo_features_check = vmxnet3_features_check, 3580 .ndo_get_stats64 = vmxnet3_get_stats64, 3581 .ndo_tx_timeout = vmxnet3_tx_timeout, 3582 .ndo_set_rx_mode = vmxnet3_set_mc, 3583 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3584 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3585 #ifdef CONFIG_NET_POLL_CONTROLLER 3586 .ndo_poll_controller = vmxnet3_netpoll, 3587 #endif 3588 }; 3589 int err; 3590 u32 ver; 3591 struct net_device *netdev; 3592 struct vmxnet3_adapter *adapter; 3593 u8 mac[ETH_ALEN]; 3594 int size; 3595 int num_tx_queues; 3596 int num_rx_queues; 3597 int queues; 3598 unsigned long flags; 3599 3600 if (!pci_msi_enabled()) 3601 enable_mq = 0; 3602 3603 #ifdef VMXNET3_RSS 3604 if (enable_mq) 3605 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3606 (int)num_online_cpus()); 3607 else 3608 #endif 3609 num_rx_queues = 1; 3610 3611 if (enable_mq) 3612 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3613 (int)num_online_cpus()); 3614 else 3615 num_tx_queues = 1; 3616 3617 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3618 max(num_tx_queues, num_rx_queues)); 3619 if (!netdev) 3620 return -ENOMEM; 3621 3622 pci_set_drvdata(pdev, netdev); 3623 adapter = netdev_priv(netdev); 3624 adapter->netdev = netdev; 3625 adapter->pdev = pdev; 3626 3627 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 3628 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 3629 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 3630 3631 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3632 if (err) { 3633 dev_err(&pdev->dev, "dma_set_mask failed\n"); 3634 goto err_set_mask; 3635 } 3636 3637 spin_lock_init(&adapter->cmd_lock); 3638 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 3639 sizeof(struct vmxnet3_adapter), 3640 DMA_TO_DEVICE); 3641 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 3642 dev_err(&pdev->dev, "Failed to map dma\n"); 3643 err = -EFAULT; 3644 goto err_set_mask; 3645 } 3646 adapter->shared = dma_alloc_coherent( 3647 &adapter->pdev->dev, 3648 sizeof(struct Vmxnet3_DriverShared), 3649 &adapter->shared_pa, GFP_KERNEL); 3650 if (!adapter->shared) { 3651 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3652 err = -ENOMEM; 3653 goto err_alloc_shared; 3654 } 3655 3656 err = vmxnet3_alloc_pci_resources(adapter); 3657 if (err < 0) 3658 goto err_alloc_pci; 3659 3660 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 3661 if (ver & (1 << VMXNET3_REV_7)) { 3662 VMXNET3_WRITE_BAR1_REG(adapter, 3663 VMXNET3_REG_VRRS, 3664 1 << VMXNET3_REV_7); 3665 adapter->version = VMXNET3_REV_7 + 1; 3666 } else if (ver & (1 << VMXNET3_REV_6)) { 3667 VMXNET3_WRITE_BAR1_REG(adapter, 3668 VMXNET3_REG_VRRS, 3669 1 << VMXNET3_REV_6); 3670 adapter->version = VMXNET3_REV_6 + 1; 3671 } else if (ver & (1 << VMXNET3_REV_5)) { 3672 VMXNET3_WRITE_BAR1_REG(adapter, 3673 VMXNET3_REG_VRRS, 3674 1 << VMXNET3_REV_5); 3675 adapter->version = VMXNET3_REV_5 + 1; 3676 } else if (ver & (1 << VMXNET3_REV_4)) { 3677 VMXNET3_WRITE_BAR1_REG(adapter, 3678 VMXNET3_REG_VRRS, 3679 1 << VMXNET3_REV_4); 3680 adapter->version = VMXNET3_REV_4 + 1; 3681 } else if (ver & (1 << VMXNET3_REV_3)) { 3682 VMXNET3_WRITE_BAR1_REG(adapter, 3683 VMXNET3_REG_VRRS, 3684 1 << VMXNET3_REV_3); 3685 adapter->version = VMXNET3_REV_3 + 1; 3686 } else if (ver & (1 << VMXNET3_REV_2)) { 3687 VMXNET3_WRITE_BAR1_REG(adapter, 3688 VMXNET3_REG_VRRS, 3689 1 << VMXNET3_REV_2); 3690 adapter->version = VMXNET3_REV_2 + 1; 3691 } else if (ver & (1 << VMXNET3_REV_1)) { 3692 VMXNET3_WRITE_BAR1_REG(adapter, 3693 VMXNET3_REG_VRRS, 3694 1 << VMXNET3_REV_1); 3695 adapter->version = VMXNET3_REV_1 + 1; 3696 } else { 3697 dev_err(&pdev->dev, 3698 "Incompatible h/w version (0x%x) for adapter\n", ver); 3699 err = -EBUSY; 3700 goto err_ver; 3701 } 3702 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 3703 3704 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 3705 if (ver & 1) { 3706 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 3707 } else { 3708 dev_err(&pdev->dev, 3709 "Incompatible upt version (0x%x) for adapter\n", ver); 3710 err = -EBUSY; 3711 goto err_ver; 3712 } 3713 3714 if (VMXNET3_VERSION_GE_7(adapter)) { 3715 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 3716 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 3717 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3718 adapter->dev_caps[0] = adapter->devcap_supported[0] & 3719 (1UL << VMXNET3_CAP_LARGE_BAR); 3720 } 3721 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 3722 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 3723 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 3724 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 3725 (1UL << VMXNET3_CAP_OOORX_COMP); 3726 } 3727 if (adapter->dev_caps[0]) 3728 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3729 3730 spin_lock_irqsave(&adapter->cmd_lock, flags); 3731 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3732 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3733 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3734 } 3735 3736 if (VMXNET3_VERSION_GE_7(adapter) && 3737 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3738 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 3739 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 3740 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 3741 } else { 3742 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 3743 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 3744 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 3745 } 3746 3747 if (VMXNET3_VERSION_GE_6(adapter)) { 3748 spin_lock_irqsave(&adapter->cmd_lock, flags); 3749 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3750 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3751 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3752 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3753 if (queues > 0) { 3754 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 3755 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 3756 } else { 3757 adapter->num_rx_queues = min(num_rx_queues, 3758 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3759 adapter->num_tx_queues = min(num_tx_queues, 3760 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3761 } 3762 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 3763 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 3764 adapter->queuesExtEnabled = true; 3765 } else { 3766 adapter->queuesExtEnabled = false; 3767 } 3768 } else { 3769 adapter->queuesExtEnabled = false; 3770 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3771 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 3772 adapter->num_rx_queues = min(num_rx_queues, 3773 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3774 adapter->num_tx_queues = min(num_tx_queues, 3775 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3776 } 3777 dev_info(&pdev->dev, 3778 "# of Tx queues : %d, # of Rx queues : %d\n", 3779 adapter->num_tx_queues, adapter->num_rx_queues); 3780 3781 adapter->rx_buf_per_pkt = 1; 3782 3783 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3784 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 3785 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 3786 &adapter->queue_desc_pa, 3787 GFP_KERNEL); 3788 3789 if (!adapter->tqd_start) { 3790 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3791 err = -ENOMEM; 3792 goto err_ver; 3793 } 3794 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 3795 adapter->num_tx_queues); 3796 3797 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 3798 sizeof(struct Vmxnet3_PMConf), 3799 &adapter->pm_conf_pa, 3800 GFP_KERNEL); 3801 if (adapter->pm_conf == NULL) { 3802 err = -ENOMEM; 3803 goto err_alloc_pm; 3804 } 3805 3806 #ifdef VMXNET3_RSS 3807 3808 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 3809 sizeof(struct UPT1_RSSConf), 3810 &adapter->rss_conf_pa, 3811 GFP_KERNEL); 3812 if (adapter->rss_conf == NULL) { 3813 err = -ENOMEM; 3814 goto err_alloc_rss; 3815 } 3816 #endif /* VMXNET3_RSS */ 3817 3818 if (VMXNET3_VERSION_GE_3(adapter)) { 3819 adapter->coal_conf = 3820 dma_alloc_coherent(&adapter->pdev->dev, 3821 sizeof(struct Vmxnet3_CoalesceScheme) 3822 , 3823 &adapter->coal_conf_pa, 3824 GFP_KERNEL); 3825 if (!adapter->coal_conf) { 3826 err = -ENOMEM; 3827 goto err_coal_conf; 3828 } 3829 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 3830 adapter->default_coal_mode = true; 3831 } 3832 3833 if (VMXNET3_VERSION_GE_4(adapter)) { 3834 adapter->default_rss_fields = true; 3835 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 3836 } 3837 3838 SET_NETDEV_DEV(netdev, &pdev->dev); 3839 vmxnet3_declare_features(adapter); 3840 3841 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 3842 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 3843 3844 if (adapter->num_tx_queues == adapter->num_rx_queues) 3845 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 3846 else 3847 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 3848 3849 vmxnet3_alloc_intr_resources(adapter); 3850 3851 #ifdef VMXNET3_RSS 3852 if (adapter->num_rx_queues > 1 && 3853 adapter->intr.type == VMXNET3_IT_MSIX) { 3854 adapter->rss = true; 3855 netdev->hw_features |= NETIF_F_RXHASH; 3856 netdev->features |= NETIF_F_RXHASH; 3857 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 3858 } else { 3859 adapter->rss = false; 3860 } 3861 #endif 3862 3863 vmxnet3_read_mac_addr(adapter, mac); 3864 dev_addr_set(netdev, mac); 3865 3866 netdev->netdev_ops = &vmxnet3_netdev_ops; 3867 vmxnet3_set_ethtool_ops(netdev); 3868 netdev->watchdog_timeo = 5 * HZ; 3869 3870 /* MTU range: 60 - 9190 */ 3871 netdev->min_mtu = VMXNET3_MIN_MTU; 3872 if (VMXNET3_VERSION_GE_6(adapter)) 3873 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 3874 else 3875 netdev->max_mtu = VMXNET3_MAX_MTU; 3876 3877 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3878 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3879 3880 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3881 int i; 3882 for (i = 0; i < adapter->num_rx_queues; i++) { 3883 netif_napi_add(adapter->netdev, 3884 &adapter->rx_queue[i].napi, 3885 vmxnet3_poll_rx_only); 3886 } 3887 } else { 3888 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 3889 vmxnet3_poll); 3890 } 3891 3892 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 3893 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 3894 3895 netif_carrier_off(netdev); 3896 err = register_netdev(netdev); 3897 3898 if (err) { 3899 dev_err(&pdev->dev, "Failed to register adapter\n"); 3900 goto err_register; 3901 } 3902 3903 vmxnet3_check_link(adapter, false); 3904 return 0; 3905 3906 err_register: 3907 if (VMXNET3_VERSION_GE_3(adapter)) { 3908 dma_free_coherent(&adapter->pdev->dev, 3909 sizeof(struct Vmxnet3_CoalesceScheme), 3910 adapter->coal_conf, adapter->coal_conf_pa); 3911 } 3912 vmxnet3_free_intr_resources(adapter); 3913 err_coal_conf: 3914 #ifdef VMXNET3_RSS 3915 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 3916 adapter->rss_conf, adapter->rss_conf_pa); 3917 err_alloc_rss: 3918 #endif 3919 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 3920 adapter->pm_conf, adapter->pm_conf_pa); 3921 err_alloc_pm: 3922 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 3923 adapter->queue_desc_pa); 3924 err_ver: 3925 vmxnet3_free_pci_resources(adapter); 3926 err_alloc_pci: 3927 dma_free_coherent(&adapter->pdev->dev, 3928 sizeof(struct Vmxnet3_DriverShared), 3929 adapter->shared, adapter->shared_pa); 3930 err_alloc_shared: 3931 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 3932 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 3933 err_set_mask: 3934 free_netdev(netdev); 3935 return err; 3936 } 3937 3938 3939 static void 3940 vmxnet3_remove_device(struct pci_dev *pdev) 3941 { 3942 struct net_device *netdev = pci_get_drvdata(pdev); 3943 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3944 int size = 0; 3945 int num_rx_queues, rx_queues; 3946 unsigned long flags; 3947 3948 #ifdef VMXNET3_RSS 3949 if (enable_mq) 3950 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3951 (int)num_online_cpus()); 3952 else 3953 #endif 3954 num_rx_queues = 1; 3955 if (!VMXNET3_VERSION_GE_6(adapter)) { 3956 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3957 } 3958 if (VMXNET3_VERSION_GE_6(adapter)) { 3959 spin_lock_irqsave(&adapter->cmd_lock, flags); 3960 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3961 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3962 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3963 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3964 if (rx_queues > 0) 3965 rx_queues = (rx_queues >> 8) & 0xff; 3966 else 3967 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3968 num_rx_queues = min(num_rx_queues, rx_queues); 3969 } else { 3970 num_rx_queues = min(num_rx_queues, 3971 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3972 } 3973 3974 cancel_work_sync(&adapter->work); 3975 3976 unregister_netdev(netdev); 3977 3978 vmxnet3_free_intr_resources(adapter); 3979 vmxnet3_free_pci_resources(adapter); 3980 if (VMXNET3_VERSION_GE_3(adapter)) { 3981 dma_free_coherent(&adapter->pdev->dev, 3982 sizeof(struct Vmxnet3_CoalesceScheme), 3983 adapter->coal_conf, adapter->coal_conf_pa); 3984 } 3985 #ifdef VMXNET3_RSS 3986 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 3987 adapter->rss_conf, adapter->rss_conf_pa); 3988 #endif 3989 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 3990 adapter->pm_conf, adapter->pm_conf_pa); 3991 3992 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3993 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 3994 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 3995 adapter->queue_desc_pa); 3996 dma_free_coherent(&adapter->pdev->dev, 3997 sizeof(struct Vmxnet3_DriverShared), 3998 adapter->shared, adapter->shared_pa); 3999 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4000 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4001 free_netdev(netdev); 4002 } 4003 4004 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4005 { 4006 struct net_device *netdev = pci_get_drvdata(pdev); 4007 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4008 unsigned long flags; 4009 4010 /* Reset_work may be in the middle of resetting the device, wait for its 4011 * completion. 4012 */ 4013 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4014 usleep_range(1000, 2000); 4015 4016 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4017 &adapter->state)) { 4018 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4019 return; 4020 } 4021 spin_lock_irqsave(&adapter->cmd_lock, flags); 4022 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4023 VMXNET3_CMD_QUIESCE_DEV); 4024 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4025 vmxnet3_disable_all_intrs(adapter); 4026 4027 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4028 } 4029 4030 4031 #ifdef CONFIG_PM 4032 4033 static int 4034 vmxnet3_suspend(struct device *device) 4035 { 4036 struct pci_dev *pdev = to_pci_dev(device); 4037 struct net_device *netdev = pci_get_drvdata(pdev); 4038 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4039 struct Vmxnet3_PMConf *pmConf; 4040 struct ethhdr *ehdr; 4041 struct arphdr *ahdr; 4042 u8 *arpreq; 4043 struct in_device *in_dev; 4044 struct in_ifaddr *ifa; 4045 unsigned long flags; 4046 int i = 0; 4047 4048 if (!netif_running(netdev)) 4049 return 0; 4050 4051 for (i = 0; i < adapter->num_rx_queues; i++) 4052 napi_disable(&adapter->rx_queue[i].napi); 4053 4054 vmxnet3_disable_all_intrs(adapter); 4055 vmxnet3_free_irqs(adapter); 4056 vmxnet3_free_intr_resources(adapter); 4057 4058 netif_device_detach(netdev); 4059 4060 /* Create wake-up filters. */ 4061 pmConf = adapter->pm_conf; 4062 memset(pmConf, 0, sizeof(*pmConf)); 4063 4064 if (adapter->wol & WAKE_UCAST) { 4065 pmConf->filters[i].patternSize = ETH_ALEN; 4066 pmConf->filters[i].maskSize = 1; 4067 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4068 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4069 4070 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4071 i++; 4072 } 4073 4074 if (adapter->wol & WAKE_ARP) { 4075 rcu_read_lock(); 4076 4077 in_dev = __in_dev_get_rcu(netdev); 4078 if (!in_dev) { 4079 rcu_read_unlock(); 4080 goto skip_arp; 4081 } 4082 4083 ifa = rcu_dereference(in_dev->ifa_list); 4084 if (!ifa) { 4085 rcu_read_unlock(); 4086 goto skip_arp; 4087 } 4088 4089 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4090 sizeof(struct arphdr) + /* ARP header */ 4091 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4092 2 * sizeof(u32); /*2 IPv4 addresses */ 4093 pmConf->filters[i].maskSize = 4094 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4095 4096 /* ETH_P_ARP in Ethernet header. */ 4097 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4098 ehdr->h_proto = htons(ETH_P_ARP); 4099 4100 /* ARPOP_REQUEST in ARP header. */ 4101 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4102 ahdr->ar_op = htons(ARPOP_REQUEST); 4103 arpreq = (u8 *)(ahdr + 1); 4104 4105 /* The Unicast IPv4 address in 'tip' field. */ 4106 arpreq += 2 * ETH_ALEN + sizeof(u32); 4107 *(__be32 *)arpreq = ifa->ifa_address; 4108 4109 rcu_read_unlock(); 4110 4111 /* The mask for the relevant bits. */ 4112 pmConf->filters[i].mask[0] = 0x00; 4113 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4114 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4115 pmConf->filters[i].mask[3] = 0x00; 4116 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4117 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4118 4119 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4120 i++; 4121 } 4122 4123 skip_arp: 4124 if (adapter->wol & WAKE_MAGIC) 4125 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4126 4127 pmConf->numFilters = i; 4128 4129 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4130 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4131 *pmConf)); 4132 adapter->shared->devRead.pmConfDesc.confPA = 4133 cpu_to_le64(adapter->pm_conf_pa); 4134 4135 spin_lock_irqsave(&adapter->cmd_lock, flags); 4136 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4137 VMXNET3_CMD_UPDATE_PMCFG); 4138 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4139 4140 pci_save_state(pdev); 4141 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4142 adapter->wol); 4143 pci_disable_device(pdev); 4144 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4145 4146 return 0; 4147 } 4148 4149 4150 static int 4151 vmxnet3_resume(struct device *device) 4152 { 4153 int err; 4154 unsigned long flags; 4155 struct pci_dev *pdev = to_pci_dev(device); 4156 struct net_device *netdev = pci_get_drvdata(pdev); 4157 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4158 4159 if (!netif_running(netdev)) 4160 return 0; 4161 4162 pci_set_power_state(pdev, PCI_D0); 4163 pci_restore_state(pdev); 4164 err = pci_enable_device_mem(pdev); 4165 if (err != 0) 4166 return err; 4167 4168 pci_enable_wake(pdev, PCI_D0, 0); 4169 4170 vmxnet3_alloc_intr_resources(adapter); 4171 4172 /* During hibernate and suspend, device has to be reinitialized as the 4173 * device state need not be preserved. 4174 */ 4175 4176 /* Need not check adapter state as other reset tasks cannot run during 4177 * device resume. 4178 */ 4179 spin_lock_irqsave(&adapter->cmd_lock, flags); 4180 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4181 VMXNET3_CMD_QUIESCE_DEV); 4182 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4183 vmxnet3_tq_cleanup_all(adapter); 4184 vmxnet3_rq_cleanup_all(adapter); 4185 4186 vmxnet3_reset_dev(adapter); 4187 err = vmxnet3_activate_dev(adapter); 4188 if (err != 0) { 4189 netdev_err(netdev, 4190 "failed to re-activate on resume, error: %d", err); 4191 vmxnet3_force_close(adapter); 4192 return err; 4193 } 4194 netif_device_attach(netdev); 4195 4196 return 0; 4197 } 4198 4199 static const struct dev_pm_ops vmxnet3_pm_ops = { 4200 .suspend = vmxnet3_suspend, 4201 .resume = vmxnet3_resume, 4202 .freeze = vmxnet3_suspend, 4203 .restore = vmxnet3_resume, 4204 }; 4205 #endif 4206 4207 static struct pci_driver vmxnet3_driver = { 4208 .name = vmxnet3_driver_name, 4209 .id_table = vmxnet3_pciid_table, 4210 .probe = vmxnet3_probe_device, 4211 .remove = vmxnet3_remove_device, 4212 .shutdown = vmxnet3_shutdown_device, 4213 #ifdef CONFIG_PM 4214 .driver.pm = &vmxnet3_pm_ops, 4215 #endif 4216 }; 4217 4218 4219 static int __init 4220 vmxnet3_init_module(void) 4221 { 4222 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4223 VMXNET3_DRIVER_VERSION_REPORT); 4224 return pci_register_driver(&vmxnet3_driver); 4225 } 4226 4227 module_init(vmxnet3_init_module); 4228 4229 4230 static void 4231 vmxnet3_exit_module(void) 4232 { 4233 pci_unregister_driver(&vmxnet3_driver); 4234 } 4235 4236 module_exit(vmxnet3_exit_module); 4237 4238 MODULE_AUTHOR("VMware, Inc."); 4239 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4240 MODULE_LICENSE("GPL v2"); 4241 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4242